Initial commit.

Author: coderanger

.gitignore

@@ -0,0 +1,3 @@
+.envrc
+.venv/
+__pycache__/

devopsdays.py

@@ -0,0 +1,97 @@
+import re
+
+import dateparser
+import requests
+from bs4 import BeautifulSoup
+
+def get(url):
+    res = requests.get(url)
+    return BeautifulSoup(res.text, 'html.parser')
+
+
+def parse_events():
+    root = get('https://www.devopsdays.org/events/')
+    for elm in root.select('.col-md-12 .row')[1].find_all('a'):
+        yield elm['href']
+
+
+def parse_open_cfps():
+    root = get('https://www.devopsdays.org/speaking/')
+    for row in root.select('table.sortable tbody tr'):
+        yield {
+            'Location': row.find('a').string,
+            'Conference URL': 'https://www.devopsdays.org' + row.find('a')['href'],
+            'CFP End Date': dateparser.parse(row.find_all('td')[1].string.strip()),
+            'Conference Start Date': dateparser.parse(row.find_all('td')[2].string.strip()).date(),
+        }
+
+
+def parse_event(url):
+    root = get(url+'welcome/')
+
+    cfp_nav = None
+    for nav in root.select('.nav-link'):
+        nav_text = str(nav.string).lower()
+        if 'propose' in nav_text or 'cfp' in nav_text:
+            cfp_nav = nav
+            break
+    if cfp_nav is None:
+        propose_elm = root.find('strong', string='Propose')
+        if propose_elm:
+            cfp_nav = propose_elm.parent.next_sibling.find('a')
+    if cfp_nav is None:
+        return None
+    cfp_url = cfp_nav['href']
+    if cfp_url.startswith('/'):
+        cfp_url = f'https://www.devopsdays.org{cfp_url}'
+
+
+    dates_elm = root.find('strong', string='Dates')
+    if dates_elm:
+        dates = dates_elm.parent.next_sibling.string.split('-')
+        event_end = dateparser.parse(dates[-1]).date()
+    else:
+        dates = root.select('.welcome-page-date')[0].contents[0]
+        # Looks like "April 9 - 10, 2019"
+        md = re.match(r'^(\S+) ([ 0-9-]+), (\d+)$', dates)
+        if md:
+            month, days, year = md.group(1, 2, 3)
+            if '-' in days:
+                start_day, end_day = days.split('-')
+            else:
+                start_day = end_day = days
+            event_end = dateparser.parse(f'{month} {end_day}, {year}').date()
+            if int(start_day) > int(end_day):
+                event_end = event_end.replace(month=event_end.month+1)
+        else:
+            raise ValueError(f'Unable to find end date in {url}')
+
+    name_parts = root.select('.welcome-page')[0].string.split()
+    name_parts[0] = name_parts[0].capitalize()
+    name = ' '.join(name_parts)
+
+    return {
+        'Conference Name': name,
+        'CFP URL': cfp_url,
+        'Conference End Date': event_end,
+        'Tags': ['devops', 'devopsdays'],
+    }
+
+
+def scrape():
+    for data in parse_open_cfps():
+        evt_data = parse_event(data['Conference URL'])
+        if evt_data is None:
+            continue
+        data.update(evt_data)
+        # Papercall is already handled.
+        if 'papercall.io' in data['CFP URL']:
+            continue
+        yield data
+
+if __name__ == '__main__':
+    # print(parse_event('https://www.devopsdays.org/events/2019-indianapolis/'))
+    # for d in parse_open_cfps():
+        # print(d)
+    for d in scrape():
+        print(d)

main.py

@@ -0,0 +1,78 @@
+import itertools
+from datetime import date, datetime
+
+import devopsdays
+import papercall
+import models
+
+def scrape_all():
+    print('Scraping Papercall')
+    yield from papercall.scrape()
+    print('Scraping Devopsdays')
+    yield from devopsdays.scrape()
+
+
+def sync_record(existing, fields):
+    # Convert any needed fields:
+    for key, value in fields.items():
+        if isinstance(value, datetime):
+            fields[key] = value.replace(microsecond=0, tzinfo=None).isoformat() + '.000Z'
+        elif isinstance(value, date):
+            fields[key] = value.isoformat()
+    if not fields.get('Conference Start Date'):
+        fields.pop('Conference Start Date')
+    if not fields.get('Conference End Date'):
+        fields.pop('Conference End Date')
+    if not fields.get('Tags'):
+        fields.pop('Tags')
+
+    # No existing verison, create it.
+    if existing is None:
+        conf = models.Conference(**fields)
+        print(f'Creating {conf}')
+        conf.save()
+    else:
+        # Check if a save is needed.
+        do_update = False
+        for key, value in fields.items():
+            existing_value = existing.get(key)
+            # Special case for tags, they need to be sorted to check.
+            if key == 'Tags' and value and existing_value:
+                if sorted(value) != sorted(existing_value):
+                    print('{} {} {}'.format(key, repr(value), repr(existing_value)))
+                    do_update = True
+                    break
+                else:
+                    continue
+
+            # Special case, none and '' are okay.
+            if value == '' and existing_value is None:
+                continue
+
+            if value != existing_value:
+                print('{} {} {}'.format(key, repr(value), repr(existing_value)))
+                do_update = True
+                break
+        if do_update:
+            print(f'Updating {existing}')
+            existing.update(fields)
+            existing.save()
+
+
+def sync_all():
+    # Fetch all the conferences into a local cache.
+    conferences = {}
+    for conf in models.Conference.fetch_all():
+        conferences[conf['CFP URL']] = conf
+
+    # Run the scrapes and syncs.
+    for fields in scrape_all():
+        sync_record(conferences.get(fields['CFP URL']), fields)
+
+
+def main():
+    sync_all()
+
+
+if __name__ == '__main__':
+    main()

models.py

@@ -0,0 +1,83 @@
+import os
+from datetime import datetime
+
+import airtable
+
+
+class AirtableModel(dict):
+    class AirtablePropety:
+        def __get__(_self, _instance, owner):
+            if not hasattr(owner, '_db'):
+                if not owner.table_name:
+                    raise ValueError(f'{owner} does not define table_name')
+                owner._db = airtable.Airtable(os.environ['AIRTABLE_BASE_KEY'], owner.table_name)
+            return owner._db
+
+    table_name = None
+    db = AirtablePropety()
+
+    def __init__(self, airtable_id=None, **fields):
+        self.airtable_id = airtable_id
+        super().__init__(fields)
+
+    @classmethod
+    def fetch(cls, **query):
+        if len(query) != 1:
+            raise ValueError(f'Invalid fetch query: {query}')
+        key, value = list(query.items())[0]
+        key = key.replace('_', ' ')
+        record = cls.db.match(key, value)
+        return cls(airtable_id=record.get('id'), **record.get('fields', {}))
+
+    @classmethod
+    def fetch_all(cls):
+        for page in cls.db.get_iter():
+            for record in page:
+                yield cls(airtable_id=record.get('id'), **record.get('fields', {}))
+
+    def save(self):
+        if self.airtable_id:
+            self.db.update(self.airtable_id, self)
+        else:
+            record = self.db.insert(self)
+            self.airtable_id = record['id']
+
+
+class Conference(AirtableModel):
+    table_name = 'Conferences'
+
+    def __str__(self):
+        label = self.get('Conference Name')
+        if not label:
+            label = self['CFP URL']
+        return f'Conference: {label}'
+
+    def save(self):
+        # If we didn't have a CFP Start Date, just assume it's today.
+        self.setdefault('CFP Start Date', str(datetime.utcnow().date()))
+
+        # Handle the tags value.
+        tags = self.pop('Tags', [])
+        try:
+            super().save()
+        finally:
+            # Restore it after the save
+            self['Tags'] = tags
+        # Update any new tags.
+        for t in tags:
+            tag = Tag.fetch(Tag=t)
+            if self.airtable_id not in tag.get('Conference', []):
+                tag['Tag'] = t
+                tag.setdefault('Conference', [])
+                tag['Conference'].append(self.airtable_id)
+                tag.save()
+        # Remove any old tags.
+        for t in self.db.get(self.airtable_id)['fields'].get('Tags', []):
+            if t not in tags:
+                tag = Tag.fetch(Tag=t)
+                tag['Conferences'].delete(self.airtable_id)
+                tag.save()
+
+
+class Tag(AirtableModel):
+    table_name = 'Conference Tags'