PhantomJSはローカルホスト上でステータス200を返しますが、ライブサーバー上では403を返します

Question

指定されたURLからHTML文書を抜き取る必要があります。私のlocalhost上でPhantom JSスクリプトはURLを正常に戻しています。しかし、私は403の禁断のステータスを取得ライブサーバー

scraper.js

var system = require('system'); 
var page = require('webpage').create(); 

$url = system.args[1]; 

page.open($url, function(status) { 


    if (status == "success") { 

     var content = page.content; 
     console.log(content); 
    } 

    phantom.exit(); 

}); 

にPhantomJSはコマンド：

phantomjs scraper.js http://www.submarino.com.br/produto/126862765/ 

スクレーパーは、他のページで正常に動作します。しかしドメインwww.submarino.com.brとwww.americanas.com.brは動作しません。私はそれがアカマイと関係があることを知っています。エラー出力を持つ応答は次のとおりです。

Response (#1, stage "start"): {"body":"","bodySize":300,"contentType":"text/html","headers":[{"name":"Server","value":"AkamaiGHost"},{"name":"Mime-Version","value":"1.0"},{"name":"Content-Type","value":"text/html"},{"name":"Content-Length","value":"300"},{"name":"Expires","value":"Wed, 10 Aug 2016 00:38:13 GMT"},{"name":"Date","value":"Wed, 10 Aug 2016 00:38:13 GMT"},{"name":"Connection","value":"close"},{"name":"Set-Cookie","value":"MobileOptOut=1; path=/; domain=submarino.com.br\nb2wChannel=INTERNET; path=/; domain=submarino.com.br"},{"name":"Vary","value":"Accept-Encoding, User-Agent"}],"id":1,"redirectURL":null,"stage":"start","status":403,"statusText":"Forbidden","time":"2016-08-10T00:38:13.540Z","url":"http://www.submarino.com.br/produto/126862765/"} 
Response (#1, stage "end"): {"contentType":"text/html","headers":[{"name":"Server","value":"AkamaiGHost"},{"name":"Mime-Version","value":"1.0"},{"name":"Content-Type","value":"text/html"},{"name":"Content-Length","value":"300"},{"name":"Expires","value":"Wed, 10 Aug 2016 00:38:13 GMT"},{"name":"Date","value":"Wed, 10 Aug 2016 00:38:13 GMT"},{"name":"Connection","value":"close"},{"name":"Set-Cookie","value":"MobileOptOut=1; path=/; domain=submarino.com.br\nb2wChannel=INTERNET; path=/; domain=submarino.com.br"},{"name":"Vary","value":"Accept-Encoding, User-Agent"}],"id":1,"redirectURL":null,"stage":"end","status":403,"statusText":"Forbidden","time":"2016-08-10T00:38:13.541Z","url":"http://www.submarino.com.br/produto/126862765/"} 

それが正常に動作した場合、それが返されます。

Response (#1, stage "start"): {"body":"","bodySize":30076,"contentType":"text/html;charset=UTF-8","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/html;charset=UTF-8"},{"name":"Server","value":"Apache-Coyote/1.1"},{"name":"X-Powered-By","value":"JSF/1.2"},{"name":"x-tid","value":"CATALOGO-0d4d336f-c0f1-4b71-9663-28fa89b5c123"},{"name":"Cache-Control","value":"max-age=1800"},{"name":"Expires","value":"Wed, 10 Aug 2016 01:10:18 GMT"},{"name":"Date","value":"Wed, 10 Aug 2016 00:40:18 GMT"},{"name":"Connection","value":"keep-alive"},{"name":"Set-Cookie","value":"MobileOptOut=1; path=/; domain=submarino.com.br\nb2wChannel=INTERNET; path=/; domain=submarino.com.br"},{"name":"Vary","value":"Accept-Encoding, User-Agent"}],"id":1,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2016-08-10T00:40:18.388Z","url":"http://www.submarino.com.br/produto/126862765/"} 
Response (#1, stage "end"): {"contentType":"text/html;charset=UTF-8","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/html;charset=UTF-8"},{"name":"Server","value":"Apache-Coyote/1.1"},{"name":"X-Powered-By","value":"JSF/1.2"},{"name":"x-tid","value":"CATALOGO-0d4d336f-c0f1-4b71-9663-28fa89b5c123"},{"name":"Cache-Control","value":"max-age=1800"},{"name":"Expires","value":"Wed, 10 Aug 2016 01:10:18 GMT"},{"name":"Date","value":"Wed, 10 Aug 2016 00:40:18 GMT"},{"name":"Connection","value":"keep-alive"},{"name":"Set-Cookie","value":"MobileOptOut=1; path=/; domain=submarino.com.br\nb2wChannel=INTERNET; path=/; domain=submarino.com.br"},{"name":"Vary","value":"Accept-Encoding, User-Agent"}],"id":1,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2016-08-10T00:40:18.390Z","url":"http://www.submarino.com.br/produto/126862765/"} 

私はhurl.itや他のcURLサービスからこのサイトをカールしようとし、彼らがURLにアクセスすることができます。私ができることはありますか？これは私を夢中にさせている！

Answer 1

おそらく、それは地理的または疑わしいIP範囲の制限です。私は今すぐURLを開こうとしましたが、ページも拒否され、アメリカのプロキシ経由でアクセスして開けました。アメリカやブラジルのプロキシを使用するだけです。

掻き落としたときにも、できるだけ近い実際のブラウザの動作を模倣することが重要ですので、私はあなたのスクリプトにユーザーエージェントとビューポートのエミュレーションを追加することをお勧めしたい：

page.viewportSize = { width: 1280, height: 800 }; 
page.settings.userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36"; 

も、エラーを購読してくださいとコンソールメッセージは、ターゲットページからのエラーやメッセージを認識します。

page.onConsoleMessage = function(msg) { 
    console.log('CONSOLE: ' + msg); 
}; 

page.onError = function (msg, trace) 
{ 
    console.log(msg); 
    trace.forEach(function(item) { 
     console.log(' ', item.file, ':', item.line); 
    }) 
}