<style>/*<link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=533180719579763849&zx=c53cd691-67ea-4e76-821a-6829e12c23d0' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=533180719579763849&zx=c53cd691-67ea-4e76-821a-6829e12c23d0' rel='stylesheet'/></noscript>
<meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/>
<meta name='google-adsense-platform-domain' content='blogspot.com'/>

<script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=ca-pub-9186430912480288&host=ca-host-pub-1556223355139109" crossorigin="anonymous"></script>

<!-- data-ad-client=ca-pub-9186430912480288 -->

</head><body>*/</style>

크롤링 탐지 우회하기 / Avoid Bot Detection

옴닉

2020년 05월 04일

IP, User-Agent, Referer

가장 중요한 것은 위의 세 가지다. 물론 가장 원초적인 기준은 크롤링을 진행하는 속도지만, 이는 당연한 것이므로 배제하고, 서버 입장에서 지금 접속한 놈이 Bot인지 아닌지 를 판단하는 기준은 "이녀석이 사람같이 행동하나?" 다

IP

하나의 IP로 계속해서 웹 페이지를 돌아다닌다. 30분, 1시간, 광고를 걸어둔 웹 서버 관리자 입장에선 아주 고마운 존재라고 생각할 수도 있지만,

같은 IP로 5시간, 6시간 째 웹 서버 페이지의 제품 목록과 제품 세부사항을 왔다갔다만 하고 있다???

User-Agent

User-Agent는 브라우저, OS 등의 정보를 최소한으로 보여주는데, 리눅스 우분투에 urllib3 이딴 거면 당연히 바꿔줘야겠지? python requests에서는 헤더로 간단히 변경할 수 있다.

Referer

Referer는 지금 웹서버에 넣은 요청 이전에 내가 어디서 들어왔는지에 대한 내용이 들어있다. 정상적인 사용자라면, 웹페이지의 호스트 도메인이 적혀져있거나 바로 이전 페이지의 url이 들어있겠지만, Bot의 경우 들어있지 않을 것이다. 관리자는 얘도 Bot임을 의심할 수 있다

<style>/*
<script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/984859869-widgets.js"></script>
<script type='text/javascript'>
window['__wavt'] = 'AOuZoY7E7S7X7UMAWsZ7gDegyexuPRua8g:1731872062328';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d533180719579763849','//blkcoding.blogspot.com/2020/05/bot-detection-avoid.html','533180719579763849');
_WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '533180719579763849', 'title': '\ube14\ub7ed \ucf54\ub529', 'url': 'https://blkcoding.blogspot.com/2020/05/bot-detection-avoid.html', 'canonicalUrl': 'https://blkcoding.blogspot.com/2020/05/bot-detection-avoid.html', 'homepageUrl': 'https://blkcoding.blogspot.com/', 'searchUrl': 'https://blkcoding.blogspot.com/search', 'canonicalHomepageUrl': 'https://blkcoding.blogspot.com/', 'blogspotFaviconUrl': 'https://blkcoding.blogspot.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': false, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': '', 'encoding': 'UTF-8', 'locale': 'ko', 'localeUnderscoreDelimited': 'ko', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22\ube14\ub7ed \ucf54\ub529 - Atom\x22 href\x3d\x22https://blkcoding.blogspot.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22\ube14\ub7ed \ucf54\ub529 - RSS\x22 href\x3d\x22https://blkcoding.blogspot.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22\ube14\ub7ed \ucf54\ub529 - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/533180719579763849/posts/default\x22 /\x3e\n\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22\ube14\ub7ed \ucf54\ub529 - Atom\x22 href\x3d\x22https://blkcoding.blogspot.com/feeds/2797054233792216467/comments/default\x22 /\x3e\n', 'meTag': '', 'adsenseClientId': 'ca-pub-9186430912480288', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': true, 'adsenseAutoAds': true, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/3d970d7188fc7945', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': '\uacf5\uc720 \ub9c1\ud06c \ub9cc\ub4e4\uae30', 'key': 'link', 'shareMessage': '\uacf5\uc720 \ub9c1\ud06c \ub9cc\ub4e4\uae30', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Facebook\uc5d0 \uacf5\uc720', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'X\uc5d0 \uacf5\uc720', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Pinterest\uc5d0 \uacf5\uc720', 'target': 'pinterest'}, {'name': '\uc774\uba54\uc77c', 'key': 'email', 'shareMessage': '\uc774\uba54\uc77c', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27ko\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': true, 'jumpLinkMessage': '\uc790\uc138\ud788 \xbb', 'pageType': 'item', 'postId': '2797054233792216467', 'pageName': '\ud06c\ub864\ub9c1 \ud0d0\uc9c0 \uc6b0\ud68c\ud558\uae30 / Avoid Bot Detection', 'pageTitle': '\ube14\ub7ed \ucf54\ub529: \ud06c\ub864\ub9c1 \ud0d0\uc9c0 \uc6b0\ud68c\ud558\uae30 / Avoid Bot Detection'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': '\uc218\uc815', 'linkCopiedToClipboard': '\ub9c1\ud06c\ub97c \ud074\ub9bd\ubcf4\ub4dc\uc5d0 \ubcf5\uc0ac\ud588\uc2b5\ub2c8\ub2e4.', 'ok': '\ud655\uc778', 'postLink': '\uae00 \ub9c1\ud06c'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': '\uc0ac\uc6a9\uc790 \uc9c0\uc815', 'isResponsive': true, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': '\ud06c\ub864\ub9c1 \ud0d0\uc9c0 \uc6b0\ud68c\ud558\uae30 / Avoid Bot Detection', 'description': ' IP, User-Agent, Referer     \uac00\uc7a5 \uc911\uc694\ud55c \uac83\uc740 \uc704\uc758 \uc138 \uac00\uc9c0\ub2e4. \ubb3c\ub860 \uac00\uc7a5 \uc6d0\ucd08\uc801\uc778 \uae30\uc900\uc740 \ud06c\ub864\ub9c1\uc744 \uc9c4\ud589\ud558\ub294 \uc18d\ub3c4\uc9c0\ub9cc, \uc774\ub294 \ub2f9\uc5f0\ud55c \uac83\uc774\ubbc0\ub85c \ubc30\uc81c\ud558\uace0, \uc11c\ubc84 \uc785\uc7a5\uc5d0\uc11c \uc9c0\uae08 \uc811\uc18d\ud55c \ub188\uc774 Bot\uc778\uc9c0 \uc544\ub2cc\uc9c0 \ub97c \ud310\ub2e8\ud558\ub294 \uae30\uc900\uc740 \x22\uc774\ub140...', 'url': 'https://blkcoding.blogspot.com/2020/05/bot-detection-avoid.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 2797054233792216467}}, {'name': 'widgets', 'data': [{'title': 'Upload Image', 'type': 'Image', 'sectionId': 'upload-image', 'id': 'Image10'}, {'title': 'Logo', 'type': 'HTML', 'sectionId': 'header-main', 'id': 'HTML10'}, {'title': 'Featured Post', 'type': 'FeaturedPost', 'sectionId': 'before-blog', 'id': 'FeaturedPost1', 'postId': '5094441260813913948'}, {'title': '\ube14\ub85c\uadf8 \uac8c\uc2dc\ubb3c', 'type': 'Blog', 'sectionId': 'blog-post', 'id': 'Blog1', 'posts': [{'id': '2797054233792216467', 'title': '\ud06c\ub864\ub9c1 \ud0d0\uc9c0 \uc6b0\ud68c\ud558\uae30 / Avoid Bot Detection', 'showInlineAds': false}], 'footerBylines': [{'regionName': 'footer1', 'items': [{'name': 'author', 'label': ''}, {'name': 'timestamp', 'label': 'yyyy\ub144 MM\uc6d4 dd\uc77c'}, {'name': 'comments', 'label': 'Comment'}]}], 'allBylineItems': [{'name': 'author', 'label': ''}, {'name': 'timestamp', 'label': 'yyyy\ub144 MM\uc6d4 dd\uc77c'}, {'name': 'comments', 'label': 'Comment'}]}, {'title': '\uc778\uae30\uae00', 'type': 'PopularPosts', 'sectionId': 'sidebar-static', 'id': 'PopularPosts10', 'posts': [{'title': '\uac04\uc9c0\ub098\ub294 \ud14d\uc2a4\ud2b8 \ub9cc\ub4e4\uae30 / \uc601\uc5b4 \ud3f0\ud2b8, \ud2b9\uc218\ubb38\uc790, \uae30\ud638', 'id': 1561632116269149504}, {'title': '\uad00\ub9ac\uc790 \uad8c\ud55c \uba85\ub839\uc5b4 \ud55c\uc904\ub85c \ub044\uae30 / UAC Disable with Command Line (CMD)', 'id': 4608231991007852337}, {'title': '\uc6f9 \ud398\uc774\uc9c0 \uc5f0\uacb0\uc2dc \x22\ub9ac\ub514\ub809\uc158 \ud69f\uc218\uac00 \ub108\ubb34 \ub9ce\uc2b5\ub2c8\ub2e4\x22 \uc624\ub958 \uc774\uc720', 'id': 9016375298288577813}, {'title': '\uc708\ub3c4\uc6b0 \uba85\ub839\uc5b4(cmd)\ub85c \uc791\uc5c5 \uc2a4\ucf00\uc974\ub7ec \ucd94\uac00, \uc218\uc815, \uc81c\uac70\ud558\uae30 SchTasks', 'id': 4799013005607202589}, {'title': '\uc708\ub3c4\uc6b0 \uba85\ub839\uc5b4(cmd)\ub85c \uc804\uc6d0 \uad00\ub9ac \uc124\uc815 \ubcc0\uacbd, \uc808\uc804 \uc2dc\uac04 \uc124\uc815 powercfg', 'id': 8634502089897115683}]}, {'title': '#Recent Post', 'type': 'HTML', 'sectionId': 'sidebar-static', 'id': 'HTML19'}, {'title': 'SVG Icons', 'type': 'HTML', 'sectionId': 'jet-options', 'id': 'HTML24'}]}]);
_WidgetManager._RegisterWidget('_ImageView', new _WidgetInfo('Image10', 'upload-image', document.getElementById('Image10'), {'resize': false}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML10', 'header-main', document.getElementById('HTML10'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_FeaturedPostView', new _WidgetInfo('FeaturedPost1', 'before-blog', document.getElementById('FeaturedPost1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'blog-post', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/165072160-lbx__ko.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_PopularPostsView', new _WidgetInfo('PopularPosts10', 'sidebar-static', document.getElementById('PopularPosts10'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML19', 'sidebar-static', document.getElementById('HTML19'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML24', 'jet-options', document.getElementById('HTML24'), {}, 'displayModeFull'));
</script>
</body>*/</style>