Open
Description
This is surely true but one weird DNS response shouldn't bring the crawler down.
ERROR Fatal exception.
Traceback (most recent call last):
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/application/app.py", line 157, in run
yield from pipeline.process()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/pipeline/pipeline.py", line 194, in process
yield from self._process_one_worker()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/pipeline/pipeline.py", line 215, in _process_one_worker
task.result()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/pipeline/pipeline.py", line 119, in process
item = yield from self.process_one(_worker_id=worker_id)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/pipeline/pipeline.py", line 103, in process_one
yield from task.process(item)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/application/tasks/download.py", line 492, in process
yield from session.app_session.factory['Processor'].process(session)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/processor/delegate.py", line 29, in process
return (yield from processor.process(item_session))
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/processor/web.py", line 92, in process
return (yield from session.process())
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/processor/web.py", line 186, in process
yield from self._process_loop()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/processor/web.py", line 245, in _process_loop
exit_early, wait_time = yield from self._fetch_one(cast(Request, self._item_session.request))
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/processor/web.py", line 268, in _fetch_one
response = yield from self._web_client_session.start()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/protocol/http/web.py", line 107, in start
response = yield from session.start(request)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/protocol/http/client.py", line 87, in start
yield from self._stream.reconnect()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/protocol/http/stream.py", line 438, in reconnect
yield from self._connection.connect()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/network/pool.py", line 375, in connect
result = yield from self._resolver.resolve(self._address[0])
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/network/dns.py", line 206, in resolve
answer = yield from self._query_dns(host, family)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/wpull/network/dns.py", line 255, in _query_dns
answer = yield from event_loop.run_in_executor(None, query)
File "/home/archivebot/.pyenv/versions/3.6.15/lib/python3.6/concurrent/futures/thread.py", line 56, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/resolver.py", line 913, in query
source_port=source_port)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/query.py", line 325, in udp
q.keyring, q.mac, ignore_trailing)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/query.py", line 271, in receive_udp
ignore_trailing=ignore_trailing)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/message.py", line 823, in from_wire
reader.read()
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/message.py", line 749, in read
self._get_section(self.message.answer, ancount)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/message.py", line 723, in _get_section
self.message.origin)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/rdata.py", line 424, in from_wire
return cls.from_wire(rdclass, rdtype, wire, current, rdlen, origin)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/rdtypes/IN/AAAA.py", line 54, in from_wire
wire[current: current + rdlen])
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/inet.py", line 78, in inet_ntop
return dns.ipv6.inet_ntoa(address)
File "/home/archivebot/.pyenv/versions/3.6.15/envs/archivebot-20230414/lib/python3.6/site-packages/dns/ipv6.py", line 39, in inet_ntoa
raise ValueError("IPv6 addresses are 16 bytes long")
ValueError: IPv6 addresses are 16 bytes long
CRITICAL Sorry, Wpull unexpectedly crashed.
CRITICAL Please report this problem to the authors at Wpull's issue tracker so it may be fixed. If you know how to program, maybe help us fix it? Thank you for helping us help you help us all.