|
| 1 | +import requests |
| 2 | +from django.contrib.auth.models import User |
| 3 | +from requests import ConnectionError |
| 4 | + |
| 5 | +from readthedocs.core.utils import trigger_build |
| 6 | +from readthedocs.projects.models import Project |
| 7 | + |
| 8 | +base_url = "http://readthedocs.org/api/v2/project/" |
| 9 | +COUNT = 0 |
| 10 | + |
| 11 | + |
| 12 | +def create_project(repo, name, repo_type): |
| 13 | + user = User.objects.all().get(username="safwan") |
| 14 | + try: |
| 15 | + p = Project.objects.create(repo=repo, name=name, repo_type=repo_type) |
| 16 | + trigger_build(p, basic=True) |
| 17 | + p.users.add(user) |
| 18 | + print(u"successfully created {}".format(p.name)) |
| 19 | + return p.id |
| 20 | + except Exception: |
| 21 | + print("unsuccessful", name) |
| 22 | + |
| 23 | + |
| 24 | +def fetch(url=None, all_projects=[]): |
| 25 | + url = url or base_url |
| 26 | + resp = requests.get(url) |
| 27 | + data = resp.json() |
| 28 | + results = data['results'] |
| 29 | + |
| 30 | + for project in results: |
| 31 | + canonical_url = project['canonical_url'] |
| 32 | + try: |
| 33 | + p = Project.objects.all().filter(name=project['name']) |
| 34 | + if not p.exists(): |
| 35 | + req = requests.get(canonical_url) |
| 36 | + if req.status_code == 200 and project['documentation_type'] == "sphinx": |
| 37 | + p = create_project(repo=project['repo'], name=project['name'], |
| 38 | + repo_type=project['repo_type']) |
| 39 | + if p: |
| 40 | + all_projects.append(p) |
| 41 | + |
| 42 | + if len(all_projects) == 10000: |
| 43 | + print("Successufl") |
| 44 | + return None |
| 45 | + except ConnectionError: |
| 46 | + pass |
| 47 | + |
| 48 | + next_url = data['next'] |
| 49 | + fetch(url=next_url) |
| 50 | + |
| 51 | + |
| 52 | +fetch(url=base_url) |
0 commit comments