Я пытаюсь очистить веб-сайт, защищенный CloudFlare, с помощью HTMLUnit
, и для этого мне нужно добавить файл cookie и пользовательский агент. Я уже добавил файл cookie с помощью CookieManager
, но у меня возникли проблемы с настройкой пользовательского агента с классом BrowserVersion
. То, что я думаю, должно произойти, должен появиться источник HTML. Это то, что я сделал:Как исправить эту BrowserVersion для HTMLUnit?
final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0";
final String COOKIE = "__cfduid=da14d952134a51b820193141bfa38717e1439243071; cf_clearance=e44e89e8388780ffdb99dbcd46d8be663183bd75-1439243075-604800; __utma=248210461.350578717.1439243076.1439243076.1439300541.2; __utmz=248210461.1439243076.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ASP.NET_SessionId=qnxzuut4sqb0mhgyxgykfp2v; __utmb=248210461.5.10.1439300541; __utmc=248210461; username=; password=; __utmt=1; MarketGidStorage=%7B%220%22%3A%7B%22svspr%22%3A%22http%3A%2F%2Fkissanime.com%2FAnime%2FJitsu-wa-Watashi-wa%2FEpisode-006%3Fid%3D113822%22%2C%22svsds%22%3A1%2C%22TejndEEDj%22%3A%22MTQzOTMwMTc1MTMxNzc5OTI0NTE%3D%22%7D%2C%22C7992%22%3A%7B%22page%22%3A1%7D%7D; __atuvc=1%7C32; __atuvs=55ca0076b32cd26c000";
WebClient webClient = new WebClient(new BrowserVersion("Firefox", "5.0 (Windows NT 10.0; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0", USER_AGENT, 39.0f));
CookieManager cookieManager = webClient.getCookieManager();
cookieManager.setCookiesEnabled(true);
cookieManager.addCookie(new Cookie("kissanime.com", "cookie", COOKIE));
webClient.setCookieManager(cookieManager);
Page page = webClient.getPage("http://www.kissanime.com");
Я знаю, что я делаю что-то неправильно, потому что это исключение брошено:
Aug 12, 2015 11:01:59 AM com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine run
SEVERE: Exception while initializing JavaScript for the page
java.lang.NullPointerException
at com.gargoylesoftware.htmlunit.javascript.HtmlUnitContextFactory.hasFeature(HtmlUnitContextFactory.java:340)
at net.sourceforge.htmlunit.corejs.javascript.Context.hasFeature(Context.java:2459)
at net.sourceforge.htmlunit.corejs.javascript.ScriptableObject.getProperty(ScriptableObject.java:2378)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.init(JavaScriptEngine.java:304)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.access$000(JavaScriptEngine.java:105)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$1.run(JavaScriptEngine.java:173)
at net.sourceforge.htmlunit.corejs.javascript.Context.call(Context.java:628)
at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.call(ContextFactory.java:513)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.initialize(JavaScriptEngine.java:184)
at com.gargoylesoftware.htmlunit.WebClient.initialize(WebClient.java:1008)
at com.gargoylesoftware.htmlunit.WebWindowImpl.setEnclosedPage(WebWindowImpl.java:135)
at com.gargoylesoftware.htmlunit.html.HTMLParser.parse(HTMLParser.java:221)
at com.gargoylesoftware.htmlunit.html.HTMLParser.parseHtml(HTMLParser.java:199)
at com.gargoylesoftware.htmlunit.DefaultPageCreator.createHtmlPage(DefaultPageCreator.java:272)
at com.gargoylesoftware.htmlunit.DefaultPageCreator.createPage(DefaultPageCreator.java:160)
at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:476)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:350)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:415)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:400)
at VideoURLScraper.main(VideoURLScraper.java:29)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:140)
Exception in thread "main" ======= EXCEPTION START ========
Exception class=[java.lang.NullPointerException]
com.gargoylesoftware.htmlunit.ScriptException
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$1.run(JavaScriptEngine.java:177)
at net.sourceforge.htmlunit.corejs.javascript.Context.call(Context.java:628)
at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.call(ContextFactory.java:513)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.initialize(JavaScriptEngine.java:184)
at com.gargoylesoftware.htmlunit.WebClient.initialize(WebClient.java:1008)
at com.gargoylesoftware.htmlunit.WebWindowImpl.setEnclosedPage(WebWindowImpl.java:135)
at com.gargoylesoftware.htmlunit.html.HTMLParser.parse(HTMLParser.java:221)
at com.gargoylesoftware.htmlunit.html.HTMLParser.parseHtml(HTMLParser.java:199)
at com.gargoylesoftware.htmlunit.DefaultPageCreator.createHtmlPage(DefaultPageCreator.java:272)
at com.gargoylesoftware.htmlunit.DefaultPageCreator.createPage(DefaultPageCreator.java:160)
at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:476)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:350)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:415)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:400)
at VideoURLScraper.main(VideoURLScraper.java:29)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:140)
Caused by: java.lang.NullPointerException
at com.gargoylesoftware.htmlunit.javascript.HtmlUnitContextFactory.hasFeature(HtmlUnitContextFactory.java:340)
at net.sourceforge.htmlunit.corejs.javascript.Context.hasFeature(Context.java:2459)
at net.sourceforge.htmlunit.corejs.javascript.ScriptableObject.getProperty(ScriptableObject.java:2378)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.init(JavaScriptEngine.java:304)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.access$000(JavaScriptEngine.java:105)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$1.run(JavaScriptEngine.java:173)
... 19 more
Enclosed exception:
java.lang.NullPointerException
at com.gargoylesoftware.htmlunit.javascript.HtmlUnitContextFactory.hasFeature(HtmlUnitContextFactory.java:340)
at net.sourceforge.htmlunit.corejs.javascript.Context.hasFeature(Context.java:2459)
at net.sourceforge.htmlunit.corejs.javascript.ScriptableObject.getProperty(ScriptableObject.java:2378)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.init(JavaScriptEngine.java:304)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.access$000(JavaScriptEngine.java:105)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$1.run(JavaScriptEngine.java:173)
at net.sourceforge.htmlunit.corejs.javascript.Context.call(Context.java:628)
at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.call(ContextFactory.java:513)
at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.initialize(JavaScriptEngine.java:184)
at com.gargoylesoftware.htmlunit.WebClient.initialize(WebClient.java:1008)
at com.gargoylesoftware.htmlunit.WebWindowImpl.setEnclosedPage(WebWindowImpl.java:135)
at com.gargoylesoftware.htmlunit.html.HTMLParser.parse(HTMLParser.java:221)
at com.gargoylesoftware.htmlunit.html.HTMLParser.parseHtml(HTMLParser.java:199)
at com.gargoylesoftware.htmlunit.DefaultPageCreator.createHtmlPage(DefaultPageCreator.java:272)
at com.gargoylesoftware.htmlunit.DefaultPageCreator.createPage(DefaultPageCreator.java:160)
at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:476)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:350)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:415)
at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:400)
at VideoURLScraper.main(VideoURLScraper.java:29)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:140)
======= EXCEPTION END ========
Какая проблема? Что происходит? Что должно произойти? – RealSkeptic
Я отредактировал вопрос, чтобы ответить на эти 3 вопроса! Я бы очень признателен за любую помощь в этом! :) – silverAndroid