Feature/codechef

codedigger/codechef/cron.py

@@ -1,20 +1,8 @@
-from .model_utils import create_or_update_codechefContest,create_or_update_codechefProblem
-from .models import CodechefContest,CodechefContestProblems
 from problem.models import Problem
-from .scraper_utils import ContestData,ProblemData,getContestDivision
-from .scraper import contestScraper,problemScraper,divisionScraper
-
-
-def OffsetLoader(contest_type):
-
-    requested_contests = []
-    for i in range(0, 60, 20):  #offset {0, 20, 40} for multiple pages of contests.
-        contests_data = contestScraper(i, contest_type)
-
-        for contests in contests_data['contests']:
-            requested_contests.append(contests)
-
-    return requested_contests
+from codechef.models import CodechefContest,CodechefContestProblems
+from codechef.scraper import contestScraper,problemScraper,divisionScraper
+from codechef.scraper_utils import ContestData,ProblemData,getContestDivision,OffsetLoader
+from codechef.model_utils import create_or_update_codechefContest,create_or_update_codechefProblem
 
 
 def update_AllContests():

codedigger/codechef/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.1.4 on 2021-10-13 10:11
+# Generated by Django 3.1.4 on 2021-11-06 16:17
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -16,36 +16,20 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='CodechefContest',
             fields=[
-                ('id',
-                 models.AutoField(auto_created=True,
-                                  primary_key=True,
-                                  serialize=False,
-                                  verbose_name='ID')),
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=200)),
                 ('contestId', models.CharField(db_index=True, max_length=10)),
                 ('duration', models.IntegerField(blank=True, null=True)),
-                ('startTime',
-                 models.CharField(blank=True, max_length=100, null=True)),
-                ('division',
-                 models.CharField(blank=True, max_length=5, null=True)),
-                ('url', models.CharField(blank=True, max_length=200,
-                                         null=True)),
+                ('startTime', models.DateTimeField(blank=True, null=True)),
+                ('url', models.CharField(blank=True, max_length=200, null=True)),
             ],
         ),
         migrations.CreateModel(
             name='CodechefContestProblems',
             fields=[
-                ('id',
-                 models.AutoField(auto_created=True,
-                                  primary_key=True,
-                                  serialize=False,
-                                  verbose_name='ID')),
-                ('contest',
-                 models.ForeignKey(on_delete=django.db.models.deletion.CASCADE,
-                                   to='codechef.codechefcontest')),
-                ('problem',
-                 models.ForeignKey(on_delete=django.db.models.deletion.CASCADE,
-                                   to='problem.problem')),
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('contest', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='codechef.codechefcontest')),
+                ('problem', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='problem.problem')),
             ],
         ),
     ]

codedigger/codechef/model_utils.py

@@ -1,34 +1,32 @@
 from datetime import datetime
-
-from .models import CodechefContest
-from .models import CodechefContest, CodechefContestProblems
 from problem.models import Problem
-from codechef.scraper import contestScraper, problemScraper, divisionScraper
-from codechef.scraper_utils import OffsetLoader, getContestDivision, ContestData, ProblemData
+from codechef.models import CodechefContest, CodechefContestProblems
+from codechef.scraper_utils import ContestData, ProblemData
 
 
 def create_or_update_codechefProblem(contestId):
     problemdata = ProblemData(contestId)
     for problem in problemdata:
-        Prob = Problem.get_or_create(name=problem['Name'],
+        Prob, created = Problem.objects.get_or_create(name=problem['Name'],
                                      prob_id=problem['ProblemCode'],
                                      url=problem['ProblemURL'],
-                                     contest_id=problem['ContestId'],
+                                     contest_id=contestId,
                                      platform=problem['Platform'])
-        cont = CodechefContest.objects.get_or_create(
-            contestId=Problem['ContestCode'], )
-        codechefProb = CodechefContestProblems.get_or_create(contest=cont,
-                                                             problem=Prob)
+        # Prob.name = problem['Name']
+        # Prob.url = problem['ProblemURL']
+        # Prob.contest_id = contestId
+        cont = CodechefContest.objects.get(contestId=problem['ContestId'])
+        codechefProb = CodechefContestProblems.objects.get_or_create(contest = cont, problem = Prob)
 
 
 def create_or_update_codechefContest(contest):
-        contestDate = datetime.strptime(contest['contest_start_date'],
+        contestDate = datetime.strptime(contest['StartTime'],
                                         "%d %B %Y  %H:%M:%S")
         cont = CodechefContest.objects.get_or_create(
             name=contest['Name'],
             contestId=contest['ContestCode'],
             duration=contest['Duration'],
-            StartTime=contestDate,
+            startTime=contestDate,
             url=contest['ContestURL'])
 
 

codedigger/codechef/models.py

@@ -17,9 +17,8 @@ def __str__(self):
 
 
 class CodechefContestProblems(models.Model):
-
-    contest = models.ForeignKey(CodechefContest, on_delete=models.CASCADE)
-    problem = models.ForeignKey(Problem, on_delete=models.CASCADE)
+    contest = models.ForeignKey(CodechefContest, blank = True,on_delete=models.CASCADE)
+    problem = models.ForeignKey(Problem, blank = True,on_delete=models.CASCADE)
 
     def __str__(self):
         return self.problem.prob_id

codedigger/codechef/scraper.py

@@ -7,7 +7,7 @@
 
 def divisionScraper(contest_id):
 
-    contest_url = "https://www.codechef.com/api/contests/" + contest_id
+    contest_url = f"https://www.codechef.com/api/contests/{contest_id}"
     contest_req = requests.get(contest_url)
     if contest_req.status_code != 200:
         raise ValidationException(
@@ -33,7 +33,7 @@ def contestScraper(offset, contest_type):
 
 def problemScraper(contest_code):
 
-    query_problem_url = f"https://www.codechef.com/api/contests/" + contest_code
+    query_problem_url = f"https://www.codechef.com/api/contests/{contest_code}"
     # Query URL might change in future.
     problem_data = requests.get(query_problem_url)
     if problem_data.status_code != 200:

codedigger/codechef/scraper_utils.py

@@ -1,10 +1,22 @@
+
+import os, json, django
 import requests
 from bs4 import BeautifulSoup
 from time import sleep
-import os, json, django
 from user.exception import ValidationException
-from codechef.scraper import contestScraper, problemScraper, divisionScraper
-from codechef.cron import OffsetLoader
+from codechef.scraper import problemScraper
+
+
+def OffsetLoader(contest_type):
+
+    requested_contests = []
+    for i in range(0, 60, 20):  #offset {0, 20, 40} for multiple pages of contests.
+        contests_data = contestScraper(i, contest_type)
+
+        for contests in contests_data['contests']:
+            requested_contests.append(contests)
+
+    return requested_contests
 
 def getContestDivision(contest_id):
 
@@ -78,6 +90,3 @@ def ProblemData(contest_code):
         all_problems.append(finalProblemData)
 
     return (all_problems)
-
-xd = ProblemData("COOK117B")
-print(xd)

codedigger/codechef/test_fixtures/model_utils_fixture.py

@@ -1,9 +1,9 @@
 codechef_contest = {
-    "Name": contest['contest_name'],
-    "ContestCode": contest_id,
-    "Duration": contest['contest_duration'],
-    "StartTime": contest_updated_date,
-    'ContestURL': "https://www.codechef.com/" + contest_id
+    "Name": "July Cook-Off 2021",
+    "ContestCode": "COOK131B",
+    "Duration": 165,
+    "StartTime": "28 July 2021  20:00:00",
+    "ContestURL": "https://www.codechef.com/COOK131B"
 }
 
 codechef_problem = [