From 9d91371b559982d53b8e55f2d991865e13efbcbf Mon Sep 17 00:00:00 2001 From: Jan Losinski Date: Mon, 21 Jan 2019 14:50:59 +0100 Subject: [PATCH] Remove multi booking in one go This never worked because the service requires a fresh session per booking. --- booker.py | 2 +- punkow/scraper.py | 30 +++++++++++++++--------------- punkow/service/worker.py | 24 ++++++++++++------------ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/booker.py b/booker.py index f67b4e6..de6eeb3 100644 --- a/booker.py +++ b/booker.py @@ -35,7 +35,7 @@ if __name__ == "__main__": logger.info("Try to get an appointment at %s", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) data = BookingData(name=args.name, email=args.email) svc = BookingService(url, debug=False) - for booked in svc.book([data]): + for booked in svc.book(data): if data == booked: break time.sleep(max(30, args.interval)) diff --git a/punkow/scraper.py b/punkow/scraper.py index b60a4cd..8d375e8 100644 --- a/punkow/scraper.py +++ b/punkow/scraper.py @@ -27,6 +27,14 @@ class BookingData(typing.NamedTuple): id: typing.Optional[int] +class BookingResult(typing.NamedTuple): + name: str + email: str + process_id: str + auth_key: str + metadata: typing.Dict[str, str] + + class BookingService(object): def __init__(self, start_url, debug=True, hide_sensitive_data=False): self.session = None @@ -53,8 +61,6 @@ class BookingService(object): self.session.headers.update({'Referrer': old_referrer}) def _print_details(self, zms, depth): - if self.sensitive: - depth = min(depth, 3) meta = {} groups = zms.find_all("div", "collapsible-toggle") for group in groups[:depth]: @@ -71,7 +77,9 @@ class BookingService(object): title = title.text.strip() desc = next(desc.stripped_strings) - logging.debug(" %s: %s", title, desc) + if not self.sensitive: + logging.debug(" %s: %s", title, desc) + meta[title] = desc return meta @@ -159,7 +167,7 @@ class BookingService(object): zms = html.find("div", {"class": "zms"}) logger.debug("Loaded booking form:") - self._print_details(zms, 4) + details = self._print_details(zms, 4) formdata = {"familyName": name, "email": email, "form_validate": "1", "agbgelesen": "1"} process = zms.find("input", {"id": "process"}) @@ -204,20 +212,12 @@ class BookingService(object): return True - def book(self, data: typing.List[BookingData]): + def book(self, data: BookingData): logging.info("Look for appointments at %s", BASE_URL + self.start_url) - data_iter = iter(data) - cur_data = next(data_iter, None) - if cur_data is None: - return - for day_url in self._iter_bookable_day_urls(self.start_url): with self._local_referrer(): for slot_url in self._iter_bookable_times(day_url): - if self._book_appointment(slot_url, cur_data.name, cur_data.email): - yield cur_data - cur_data = next(data_iter, None) - if cur_data is None: - return + if self._book_appointment(slot_url, data.name, data.email): + yield data return diff --git a/punkow/service/worker.py b/punkow/service/worker.py index 9088bca..3cd35b7 100644 --- a/punkow/service/worker.py +++ b/punkow/service/worker.py @@ -23,8 +23,8 @@ class _WorkerRequest(typing.NamedTuple): class RequestQueue(object): def __init__(self): - self._targets = [] - self._requests = [] + self._targets = [] # type: typing.List[str] + self._requests = [] # type: typing.List[_WorkerRequest] def enqueue(self, req: model.Request): if req.target not in self._requests: @@ -33,9 +33,9 @@ class RequestQueue(object): self._requests.append( _WorkerRequest(req.id, req.data.name, req.data.email, req.target)) - def iterate(self): + def iterate(self) -> typing.Generator[_WorkerRequest, None, None]: for item in self._requests: - yield item.target, [item] + yield item def is_empty(self): return len(self._targets) == 0 @@ -49,13 +49,13 @@ class RequestQueue(object): return len(self._requests) -def _book(target: str, reqs: typing.List[_WorkerRequest], debug=False) -> typing.List[int]: - data = [scraper.BookingData(name=req.name, email=req.email, id=req.id) - for req in reqs] - if target.startswith(scraper.BASE_URL): - target = target[len(scraper.BASE_URL):] +def _book(req: _WorkerRequest, debug=False) -> typing.List[int]: + data = scraper.BookingData(name=req.name, email=req.email, id=req.id) + target = req.target + if req.target.startswith(scraper.BASE_URL): + target = req.target[len(scraper.BASE_URL):] - logger.debug("Try to book %d appointments for %s", len(reqs), target) + logger.debug("Try to book one appointment for %s", target) booked_ids = [] try: @@ -140,10 +140,10 @@ class Worker(object): return booked = [] - for target, reqs in requests.iterate(): + for req in requests.iterate(): booked.extend( await self._loop.run_in_executor(self._executor, - functools.partial(_book, target, reqs, debug=self._debug))) + functools.partial(_book, req, debug=self._debug))) await self._loop.run_in_executor(None, functools.partial(self.cleanup_booked, booked)) await self._loop.run_in_executor(None, self.cleanup_old)