Merge pull request #485 from sparcs-kaist/feature/meal_crawler

New Feature : 학식 API

Author: Qndndn

apps/core/management/scripts/meal_crawler.py

@@ -1,76 +1,93 @@
-import time
-from collections import defaultdict
-
-from apps.core.management.scripts.reminder_email_for_reply import send_email
-from apps.core.models import BestArticle
-from apps.kaist.portal.worker import Worker as PortalCrawlWorker
-from ara import celery_app, redis
-
-
-@celery_app.task
-def crawl_portal():
-    PortalCrawlWorker.fetch_and_save_from_the_latest(batch_size=32)
-
-
-def _get_redis_key(type_):
-    return f"articles:{type_}"
-
-
-def _get_best(days, period):
-    BestArticle.objects.filter(latest=True, period=period).update(latest=False)
-
-    type_ = "vote"
-    to_ts = time.time()
-    from_ts = to_ts - 24 * 60 * 60 * days
-
-    vote_objs = redis.get_objs_by_values(_get_redis_key(type_), f"({from_ts}", to_ts)
-
-    article_votes = defaultdict(int)
-    for obj in vote_objs:
-        article_id, vote, _, _ = obj.split(":")
-        article_votes[article_id] += int(vote)
-
-    type_ = "hit"
-    hit_objs = redis.get_objs_by_values(_get_redis_key(type_), f"({from_ts}", to_ts)
-
-    article_hits = defaultdict(int)
-    for obj in hit_objs:
-        article_id, hit, _, _ = obj.split(":")
-        article_hits[article_id] += int(hit)
-
-    hit_sorted = sorted(
-        article_votes.items(), key=lambda x: article_hits[x[0]], reverse=True
-    )
-    articles = []
-    keys = []
-
-    length = len(article_votes)
-    for key, _ in sorted(hit_sorted, key=lambda x: x[1], reverse=True)[:5]:
-        articles.append(BestArticle(period=period, article_id=key, latest=True))
-        keys.append(key)
-
-    if length < 5:
-        for key, _ in sorted(article_hits.items(), key=lambda x: x[1], reverse=True):
-            if key not in keys:
-                articles.append(BestArticle(period=period, article_id=key, latest=True))
-                keys.append(key)
-
-            if len(articles) >= 5:
-                break
-
-    return BestArticle.objects.bulk_create(articles)
-
-
-@celery_app.task
-def save_daily_best():
-    return _get_best(1, BestArticle.PERIOD_CHOICES_DAILY)
-
-
-@celery_app.task
-def save_weekly_best():
-    return _get_best(7, BestArticle.PERIOD_CHOICES_WEEKLY)
-
-
-@celery_app.task
-def send_email_for_reply_reminder():
-    send_email()
+import time
+from collections import defaultdict
+
+from apps.core.management.scripts.portal_crawler import crawl_hour, crawl_view
+from apps.core.management.scripts.reminder_email_for_reply import send_email
+from apps.core.management.scripts.meal_crawler import crawl_daily_meal
+from apps.core.models import BestArticle
+from ara import celery_app, redis
+
+from datetime import datetime, timedelta
+
+@celery_app.task
+def crawl_portal():
+    crawl_view()
+    crawl_hour()
+
+
+def _get_redis_key(type_):
+    return f"articles:{type_}"
+
+
+def _get_best(days, period):
+    BestArticle.objects.filter(latest=True, period=period).update(latest=False)
+
+    type_ = "vote"
+    to_ts = time.time()
+    from_ts = to_ts - 24 * 60 * 60 * days
+
+    vote_objs = redis.get_objs_by_values(_get_redis_key(type_), f"({from_ts}", to_ts)
+
+    article_votes = defaultdict(int)
+    for obj in vote_objs:
+        article_id, vote, _, _ = obj.split(":")
+        article_votes[article_id] += int(vote)
+
+    type_ = "hit"
+    hit_objs = redis.get_objs_by_values(_get_redis_key(type_), f"({from_ts}", to_ts)
+
+    article_hits = defaultdict(int)
+    for obj in hit_objs:
+        article_id, hit, _, _ = obj.split(":")
+        article_hits[article_id] += int(hit)
+
+    hit_sorted = sorted(
+        article_votes.items(), key=lambda x: article_hits[x[0]], reverse=True
+    )
+    articles = []
+    keys = []
+
+    length = len(article_votes)
+    for key, _ in sorted(hit_sorted, key=lambda x: x[1], reverse=True)[:5]:
+        articles.append(BestArticle(period=period, article_id=key, latest=True))
+        keys.append(key)
+
+    if length < 5:
+        for key, _ in sorted(article_hits.items(), key=lambda x: x[1], reverse=True):
+            if key not in keys:
+                articles.append(BestArticle(period=period, article_id=key, latest=True))
+                keys.append(key)
+
+            if len(articles) >= 5:
+                break
+
+    return BestArticle.objects.bulk_create(articles)
+
+
+@celery_app.task
+def save_daily_best():
+    return _get_best(1, BestArticle.PERIOD_CHOICES_DAILY)
+
+
+@celery_app.task
+def save_weekly_best():
+    return _get_best(7, BestArticle.PERIOD_CHOICES_WEEKLY)
+
+
+@celery_app.task
+def send_email_for_reply_reminder():
+    send_email()
+
+#학식 크롤링
+@celery_app.task
+def crawl_meal():
+    #현재 날짜로 부터 앞으로 일주일간 식단 크롤링
+    # 현재 날짜를 가져오기
+    current_date = datetime.now()
+
+    # 앞 뒤 7일간의 날짜 리스트 생성
+    dates = [(current_date + timedelta(days=i)).strftime("%Y-%m-%d") for i in range(-7, 7)]
+    for date in dates:
+        #식단 크롤링
+        crawl_daily_meal(date)
+        time.sleep(2)

apps/core/management/tasks.py

@@ -1,70 +1,78 @@
-from rest_framework import routers
-
-from apps.core.views import viewsets
-
-router = routers.DefaultRouter()
-
-# BoardViewSet
-router.register(
-    prefix=r"boards",
-    viewset=viewsets.BoardViewSet,
-)
-
-router.register(
-    prefix=r"board_groups",
-    viewset=viewsets.BoardGroupViewSet,
-)
-
-# ArticleViewSet
-router.register(
-    prefix=r"articles",
-    viewset=viewsets.ArticleViewSet,
-)
-
-# CommentViewSet
-router.register(
-    prefix=r"comments",
-    viewset=viewsets.CommentViewSet,
-)
-
-# ReportViewSet
-router.register(
-    prefix=r"reports",
-    viewset=viewsets.ReportViewSet,
-)
-
-# BlockViewSet
-router.register(
-    prefix=r"blocks",
-    viewset=viewsets.BlockViewSet,
-)
-
-# AttachmentViewSet
-router.register(
-    prefix=r"attachments",
-    viewset=viewsets.AttachmentViewSet,
-)
-
-# ScrapViewSet
-router.register(
-    prefix=r"scraps",
-    viewset=viewsets.ScrapViewSet,
-)
-
-# NotificationViewSet
-router.register(
-    prefix=r"notifications",
-    viewset=viewsets.NotificationViewSet,
-)
-
-# FAQViewSet
-router.register(
-    prefix=r"faqs",
-    viewset=viewsets.FAQViewSet,
-)
-
-# BestSearchViewSet
-router.register(
-    prefix=r"best_searches",
-    viewset=viewsets.BestSearchViewSet,
-)
+from rest_framework import routers
+
+from apps.core.views import viewsets
+
+router = routers.DefaultRouter()
+
+# BoardViewSet
+router.register(
+    prefix=r"boards",
+    viewset=viewsets.BoardViewSet,
+)
+
+router.register(
+    prefix=r"board_groups",
+    viewset=viewsets.BoardGroupViewSet,
+)
+
+# ArticleViewSet
+router.register(
+    prefix=r"articles",
+    viewset=viewsets.ArticleViewSet,
+)
+
+# CommentViewSet
+router.register(
+    prefix=r"comments",
+    viewset=viewsets.CommentViewSet,
+)
+
+# ReportViewSet
+router.register(
+    prefix=r"reports",
+    viewset=viewsets.ReportViewSet,
+)
+
+# BlockViewSet
+router.register(
+    prefix=r"blocks",
+    viewset=viewsets.BlockViewSet,
+)
+
+# AttachmentViewSet
+router.register(
+    prefix=r"attachments",
+    viewset=viewsets.AttachmentViewSet,
+)
+
+# ScrapViewSet
+router.register(
+    prefix=r"scraps",
+    viewset=viewsets.ScrapViewSet,
+)
+
+# NotificationViewSet
+router.register(
+    prefix=r"notifications",
+    viewset=viewsets.NotificationViewSet,
+)
+
+# FAQViewSet
+router.register(
+    prefix=r"faqs",
+    viewset=viewsets.FAQViewSet,
+)
+
+# BestSearchViewSet
+router.register(
+    prefix=r"best_searches",
+    viewset=viewsets.BestSearchViewSet,
+)
+
+#meal : 학식 크롤링 기능
+router.register(
+    prefix=r"meals",
+    viewset=viewsets.MealViewSet,
+    #단순한 api여서 queryset을 사용하지 않았으므로 basename 설정.
+    basename="meal",
+)

apps/core/views/router.py

@@ -1,11 +1,12 @@
-from .article import *
-from .attachment import *
-from .best_search import *
-from .block import *
-from .board import *
-from .board_group import *
-from .comment import *
-from .faq import *
-from .notification import *
-from .report import *
-from .scrap import *
+from .article import *
+from .attachment import *
+from .best_search import *
+from .block import *
+from .board import *
+from .board_group import *
+from .comment import *
+from .faq import *
+from .notification import *
+from .report import *
+from .scrap import *
+from .meal import *