Merge pull request #136 from monkey-team-3801/course-database-populate

Scripts for fetching courses implemented

package.json

@@ -65,6 +65,7 @@
         "postcss-normalize": "8.0.1",
         "postcss-preset-env": "6.7.0",
         "postcss-safe-parser": "4.0.1",
+        "qs": "^6.9.4",
         "react": "^16.13.1",
         "react-app-polyfill": "^1.0.6",
         "react-bootstrap": "^1.3.0",
@@ -107,6 +108,7 @@
         "build": "npm run build-client && npm run build-server",
         "build-client": "node scripts/build.js",
         "build-server": "webpack --config ./config/webpackServer.config.js",
+        "fetch-course": "node scripts/fetchCourse.js",
         "test": "node scripts/test.js",
         "prettier": "prettier --write .",
         "prettier-check": "prettier --check .",

scripts/fetchCourse.js

@@ -0,0 +1,141 @@
+const axios = require("axios");
+const qs = require("qs");
+const dotenv = require("dotenv");
+const _ = require("lodash");
+const fns = require("date-fns");
+// const util = require("util");
+const MongoClient = require("mongodb").MongoClient;
+
+dotenv.config();
+
+const dayToISO = {
+    Mon: 1,
+    Tue: 2,
+    Wed: 3,
+    Thu: 4,
+    Fri: 5,
+    Sat: 6,
+    Sun: 7,
+};
+
+const searchTerms = ["DECO", "CSSE", "COMP", "ENGG"];
+
+const defaultData = {
+    semester: "S2",
+    campus: "STLUC",
+    faculty: "EAIT",
+    type: ["Lecture", "Practical", "Studio", "Tutorial"],
+    days: ["1", "2", "3", "4", "5"],
+    "start-time": "00:00",
+    "end-time": "23:00",
+};
+
+const defaultConfig = {
+    method: "post",
+    url: "https://timetable.my.uq.edu.au/even/rest/timetable/subjects",
+    headers: {
+        Accept: "application/json, text/javascript, */*; q=0.01",
+        DNT: "1",
+        "X-Requested-With": "XMLHttpRequest",
+        "User-Agent":
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36",
+        "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+        "Sec-Fetch-Site": "same-origin",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Dest": "empty",
+    },
+};
+
+const getCourses = async () => {
+    let finalResponse = {};
+    for (const searchTerm of searchTerms) {
+        const data = qs.stringify({
+            ...defaultData,
+            "search-term": searchTerm,
+        });
+        const config = { ...defaultConfig, data };
+        const response = await axios(config);
+        finalResponse = { ...finalResponse, ...response.data };
+    }
+    return finalResponse;
+};
+
+const parseCourse = (courses) => {
+    const cleanedCourses = _.pickBy(
+        courses,
+        (course) =>
+            !course.subject_code.includes("_EX") && // Remove external courses
+            !course.description.toLowerCase().includes("special topics") // Remove special topics courses
+    );
+    const courseFinal = [];
+    for (const course of _.values(cleanedCourses)) {
+        const newEntry = {
+            code: course.subject_code.substring(0, 8),
+            description: course.description,
+            activities: _.values(course.activities)
+                .filter((activity) => activity.activity_type !== "Delayed")
+                .map((activity) => {
+                    const time = fns.parse(
+                        activity.start_time,
+                        "HH:mm",
+                        new Date()
+                    );
+                    return {
+                        type: activity.activity_type,
+                        code: activity.activity_code,
+                        time: time.getHours() + time.getMinutes() / 60,
+                        startDate: fns.parse(
+                            activity.start_date,
+                            "dd/MM/y",
+                            new Date()
+                        ),
+                        duration: Number(activity.duration),
+                        dayOfWeek: dayToISO[activity.day_of_week],
+                        weeks: activity.week_pattern
+                            .split("")
+                            .map((char) => Number(char)),
+                    };
+                }),
+            announcements: [],
+        };
+        courseFinal.push(newEntry);
+    }
+    return courseFinal;
+};
+
+// getCourses().then(courses => console.log(parseCourse(courses)));
+MongoClient.connect(
+    process.env.MONGODB_URI,
+    {
+        useUnifiedTopology: true,
+        useNewUrlParser: true,
+    },
+    async (err, client) => {
+        if (err) throw err;
+        let dbName;
+        // TODO: hardcoded db name
+        if (process.env.NODE_ENV === "production") {
+            dbName = "monkprod";
+        } else {
+            dbName = "test";
+        }
+        const db = client.db(dbName);
+        const courses = parseCourse(await getCourses());
+        const courseCollection = db.collection("courses", (error) => {
+            if (error) {
+                throw error;
+            }
+        });
+        const deleteResult = await courseCollection.deleteMany({});
+        console.log(
+            "Cleared course db, deleted",
+            deleteResult.deletedCount,
+            "entries."
+        );
+        const result = await courseCollection.insertMany(courses);
+        console.log("Added", result.insertedCount, "new entries.");
+        await client.close();
+    }
+);
+// getCourses().then((res) => console.log(util.inspect(parseCourse(res), false,
+// null, true)));

src/types.ts

@@ -55,7 +55,7 @@ export type CourseActivityResponseType = CourseActivityUnique & {
     time: number;
     startDate: Date;
     duration: number;
-    day_of_week: 1 | 2 | 3 | 4 | 5 | 6 | 7; // Monday->Sunday in ISO week
+    dayOfWeek: 1 | 2 | 3 | 4 | 5 | 6 | 7; // Monday->Sunday in ISO week
     weeks: Array<1 | 0>;
 };