Merge pull request #27 from CheatSOL/develop

2주차 Develop 브랜치 main으로 머지

Author: gariguri

.github/workflows/main.yml

@@ -70,8 +70,15 @@ jobs:
             sudo npm install -g pm2
             # Navigate to the application directory
             cd /home/ec2-user/CheatSOL-BE
+
+            # Install Python and pip
+            sudo yum install -y python3
+            sudo alternatives --set python /usr/bin/python3
+            sudo yum install -y python3-pip
+
             # Install project dependencies
             npm install --production
+            pip install -r requirements.txt
             # Restart or start the application with pm2
             pm2 stop all
             pm2 delete all

.gitignore

@@ -130,3 +130,4 @@ dist
 .pnp.*
 
 package-lock.json
+.idea/

app.js

@@ -3,19 +3,43 @@ var express = require("express");
 var path = require("path");
 var cookieParser = require("cookie-parser");
 var logger = require("morgan");
-
+const {wsdata}=require("./src/utils/WSPrice");
 var indexRouter = require("./src/routes/index");
-var usersRouter = require("./src/routes/users");
+var companyRouter = require("./src/routes/company");
+var googleRouter = require("./src/routes/google");
+var stockInfoRouter = require("./src/routes/stock.info.detail");
 const db = require("./src/models/DB");
-
+const http=require('http');
 var app = express();
-
 app.use(logger("dev"));
 app.use(express.json());
 app.use(express.urlencoded({ extended: false }));
 app.use(cookieParser());
+const server = http.createServer(app);
+const WebSocket = require('ws');
+const wss = new WebSocket.Server({ server });
+wss.on('connection',  async function connection(ws) {
+  console.log('새로운 WebSocket 클라이언트가 연결되었습니다.');
+  let id;
+  ws.on('message', (message) => {
+    const data = JSON.parse(message);
+    console.log('Received data:', data);
+    id = data.id;
+    if (id) {
+        wsdata(ws, id);  // id가 설정된 후에 wsdata 호출
+    } else {
+        console.error('ID is undefined');
+    }
+});
+
+  ws.on('close', function close() {
+      console.log('WebSocket 연결이 종료되었습니다.');
+  });
+});
+server.listen(3002, () => {
+  console.log('서버가 3002번 포트에서 실행 중입니다.');
+});
 
-// 데이터베이스 연결 확인 및 동기화
 db.sequelize
   .authenticate()
   .then(() => {
@@ -32,11 +56,14 @@ db.sequelize
   });
 
 app.use("/api", indexRouter);
-app.use("/users", usersRouter);
+app.use("/api/company", companyRouter);
+app.use("/api/trends", googleRouter);
+app.use("/api/stockInfo", stockInfoRouter);
 
 // catch 404 and forward to error handler
 app.use(function (req, res, next) {
-  next(createError(404));
+  createError(404);
+  res.json({ code: 404, message: "서버에 url과 일치하는 api가 없습니다." });
 });
 
 // error handler
@@ -47,7 +74,7 @@ app.use(function (err, req, res, next) {
 
   // render the error page
   res.status(err.status || 500);
-  res.render("error");
+  res.json(res.locals);
 });
 
 module.exports = app;

package.json

@@ -8,15 +8,22 @@
   },
   "dependencies": {
     "axios": "^1.7.2",
+    "body-parser": "^1.20.2",
+    "cheerio": "^1.0.0-rc.12",
     "cookie-parser": "~1.4.4",
     "debug": "~2.6.9",
     "dotenv": "^16.4.5",
     "ejs": "~2.6.1",
     "express": "~4.16.1",
+    "google-trends-api": "^4.9.2",
     "http-errors": "~1.6.3",
+    "https": "^1.0.0",
     "morgan": "~1.9.1",
+    "nodemon": "^3.1.3",
     "pg": "^8.12.0",
     "pg-hstore": "^2.3.4",
-    "sequelize": "^6.37.3"
+    "python-shell": "^5.0.0",
+    "sequelize": "^6.37.3",
+    "ws": "^8.17.1"
   }
 }

requirements.txt

@@ -0,0 +1,6 @@
+psycopg2-binary==2.9.9
+gensim==4.3.2
+pandas==2.2.2
+psycopg2==2.9.9
+python-dotenv==1.0.1
+scipy==1.10.0

src/controllers/AuthController.js

@@ -1,10 +1,48 @@
-const AuthToken = require("../models/AuthToken");
+const db = require("../models/DB");
 
-async function getAuthToken(ApiHost) {
+async function getAuthToken(apiHost) {
   try {
-    const authTokens = await AuthToken.find(ApiHost);
-    authTokens.forEach((token) => console.log(token.toJSON()));
+    const authToken = await db.AuthToken.findOne({
+      where: { name: apiHost },
+      order: [['updatedAt', 'DESC']]
+    });
+    
+    return authToken.dataValues;
   } catch (error) {
-    console.error("Error fetching AuthTokens:", error);
+    return null;
   }
 }
+
+async function setAuthToken(
+  apiHost,
+  accessToken,
+  tokenExpired,
+  tokenType = "Bearer"
+) {
+  await db.AuthToken.create({
+    name: apiHost,
+    access_token: accessToken,
+    access_token_expired: tokenExpired,
+    token_type: tokenType,
+  });
+}
+
+async function updateAuthToken(
+  apiHost,
+  accessToken,
+  tokenExpired,
+  tokenType = "Bearer"
+) {
+  await db.AuthToken.update(
+    {
+      access_token: accessToken,
+      access_token_expired: tokenExpired,
+      token_type: tokenType,
+    },
+    {
+      where: { name: apiHost },
+    }
+  );
+}
+
+module.exports = { getAuthToken, setAuthToken, updateAuthToken };

src/controllers/CompanyController.js

@@ -0,0 +1,35 @@
+const db = require("../models/DB");
+
+async function getCompanyByCode(code) {
+  try {
+    const company = await db.Company.findOne({ where: { code: code } });
+    console.log(company);
+    return company.dataValues;
+  } catch (error) {
+    console.log("회사 정보가 없음");
+    return null;
+  }
+}
+
+async function getCompanies() {
+  try {
+    const companies = await db.Company.findAll();
+    return companies;
+  } catch (error) {
+    console.log("회사 정보가 없음");
+    return null;
+  }
+}
+
+async function setCompanies(companies) {
+  await db.Company.destroy({ where: {} });
+  const companyData = companies.map((company) => ({
+    name: company.name,
+    code: company.code,
+  }));
+
+  console.log(companyData);
+  await db.Company.bulkCreate(companyData);
+}
+
+module.exports = { setCompanies, getCompanies, getCompanyByCode };

src/controllers/NewsCrawling.js

@@ -0,0 +1,98 @@
+const axios=require('axios');
+const cheerio=require('cheerio');
+const { Model, Op } = require('sequelize');
+const { CompanyNews, Company } = require('../models/DB');
+const fetchNews = require('../utils/NaverStockNews');
+
+async function fetchNewsContent(link) {
+  try {
+    const response = await axios.get(link);
+    const $ = cheerio.load(response.data);
+    const content =
+      $("#newsct_article").text().trim() ||
+      $("._article_content").text().trim();
+    return content;
+  } catch (error) {
+    console.error(`Error fetching news content from ${link}:`, error);
+    return null;
+  }
+}
+
+async function saveCodes() {
+  const companies = await Company.findAll();
+  const companyCodes = companies.map((company) => ({
+    code: company.dataValues.code,
+    id: company.dataValues.id,
+  }));
+  return companyCodes;
+}
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function saveNewsToDatabase(id, newsItems) {
+  const newsData = [];
+
+  //console.log("newsItems", newsItems);
+  for (let items of newsItems) {
+    for (let item of items) {
+      const link = `https://n.news.naver.com/article/${item.officeId}/${item.articleId}`;
+
+      // 1초 지연 (1000ms)
+      await delay(100);
+
+      try {
+        const detailedContent = await fetchNewsContent(link);
+
+        newsData.push({
+          title: item.title,
+          content: detailedContent || item.body,
+          link: link,
+          company_id: id,
+        });
+      } catch (error) {
+        console.error(`Error fetching news content for ${link}:`, error);
+        // 에러 처리 (예: 특정 뉴스 아이템에 대한 실패를 로그로 남기거나, 다시 시도할 수 있도록 로직 추가)
+      }
+    }
+  }
+  // 데이터베이스에 저장
+  if (newsData.length > 0) {
+    await CompanyNews.bulkCreate(newsData);
+  }
+}
+
+// Express 컨트롤러 함수
+async function handleNewsDatas(data, res) {
+  try {
+    // fetchNews 함수로 뉴스 아이템들 가져오기
+    const newsItems = await fetchNews(data.code);
+
+    // 가져온 뉴스 아이템들을 데이터베이스에 저장하기
+    await saveNewsToDatabase(data.id, newsItems);
+  } catch (error) {
+    console.error(`Error handling news data for company ${data.id}:`, error);
+    // 에러 처리 (예: 특정 회사에 대한 실패를 로그로 남기거나, 다시 시도할 수 있도록 로직 추가)
+  }
+}
+
+const handleCompanyNews = async (req, res, next) => {
+  try {
+    // 회사 코드를 데이터베이스에서 가져오기
+    const companyCodes = await saveCodes();
+
+    // 각 회사 코드에 대해 뉴스 아이템을 가져오고 데이터베이스에 저장하기
+    for (const data of companyCodes) {
+      await handleNewsDatas(data, res);
+    }
+
+    res
+      .status(200)
+      .json({ message: `Successfully fetched and saved news items.` });
+  } catch (error) {
+    console.error("Error handling news data:", error);
+    // 에러 응답 보내기
+    res.status(500).json({ error: "Failed to fetch and save news items." });
+  }
+};
+
+module.exports = handleCompanyNews;

src/controllers/Word2VecController.js

@@ -0,0 +1,41 @@
+const { PythonShell } = require("python-shell");
+
+require("dotenv").config(); // dotenv 설정 불러오기
+
+async function runPythonScript(scriptPath, options) {
+  console.log("Running Python Script..."); // Initial log
+  return new Promise((resolve, reject) => {
+    PythonShell.run(scriptPath, options)
+      .then((results) => {
+        try {
+          const parsedResults = JSON.parse(results.join(""));
+          resolve(parsedResults);
+        } catch (parseError) {
+          reject(parseError);
+        }
+      })
+      .catch((err) => {
+        console.error(err);
+      });
+  });
+}
+
+async function getSimilarityCompanies(word) {
+  console.log("word", word);
+  try {
+    const options = {
+      mode: "text",
+      pythonOptions: ["-u"],
+      scriptPath: "src/scripts",
+      args: [word],
+      env: process.env,
+    };
+    const results = await runPythonScript("get_similar_words.py", options);
+
+    return results;
+  } catch (error) {
+    throw new Error(error);
+  }
+}
+
+module.exports = { getSimilarityCompanies };