jan
/
punkow
mirror of https://github.com/janLo/punkow
1
0
Fork 0
Code Issues Releases Wiki Activity

Adopt scraper changes in worker

This commit is contained in:
Jan Losinski 2019-01-21 16:03:44 +01:00
parent 25be5e6cea
commit ad61ff6671
1 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
punkow/service/worker.py
View File

@ -49,25 +49,26 @@ class RequestQueue(object):
return len(self._requests) return len(self._requests)
def _book(req: _WorkerRequest, debug=False) -> typing.List[int]: def _book(req: _WorkerRequest, debug=False) -> typing.Dict[int, scraper.BookingResult]:
data = scraper.BookingData(name=req.name, email=req.email, id=req.id) data = scraper.BookingData(name=req.name, email=req.email)
target = req.target target = req.target
if req.target.startswith(scraper.BASE_URL): if req.target.startswith(scraper.BASE_URL):
target = req.target[len(scraper.BASE_URL):] target = req.target[len(scraper.BASE_URL):]
logger.debug("Try to book one appointment for %s", target) logger.debug("Try to book one appointment for %s", target)
booked_ids = [] bookings = {} # type: typing.Dict[int, scraper.BookingResult]
try: try:
svc = scraper.BookingService(target, debug=debug, hide_sensitive_data=True) svc = scraper.BookingService(target, debug=debug, hide_sensitive_data=True)
for booked in svc.book(data): booked = svc.book(data)
booked_ids.append(booked.id) if booked is not None:
bookings[req.id] = booked
except: except:
logger.exception("Exception while booking") logger.exception("Exception while booking")
logger.info("Booked %d appointments for %s", len(booked_ids), target) logger.info("Booked an appointments for %s", target)
return booked_ids return bookings
class Worker(object): class Worker(object):
@ -141,9 +142,9 @@ class Worker(object):
booked = [] booked = []
for req in requests.iterate(): for req in requests.iterate():
booked.extend( booking = await self._loop.run_in_executor(self._executor,
await self._loop.run_in_executor(self._executor, functools.partial(_book, req, debug=self._debug))
functools.partial(_book, req, debug=self._debug))) booked.extend(booking)
await self._loop.run_in_executor(None, functools.partial(self.cleanup_booked, booked)) await self._loop.run_in_executor(None, functools.partial(self.cleanup_booked, booked))
await self._loop.run_in_executor(None, self.cleanup_old) await self._loop.run_in_executor(None, self.cleanup_old)