Added CRON job to fetch data

bun.lock

@@ -3,6 +3,9 @@
   "workspaces": {
     "": {
       "name": "hc-app",
+      "dependencies": {
+        "cron": "^4.1.0",
+      },
       "devDependencies": {
         "@eslint/compat": "^1.2.5",
         "@eslint/js": "^9.18.0",
@@ -12,6 +15,7 @@
         "@tailwindcss/forms": "^0.5.9",
         "@tailwindcss/typography": "^0.5.15",
         "@tailwindcss/vite": "^4.0.0",
+        "@types/cron": "^2.4.3",
         "@types/fuzzy-search": "^2.1.5",
         "bun-types": "^1.2.4",
         "daisyui": "^5.0.0",
@@ -217,12 +221,16 @@
 
     "@types/cookie": ["@types/cookie@0.6.0", "", {}, "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA=="],
 
+    "@types/cron": ["@types/cron@2.4.3", "", { "dependencies": { "cron": "*" } }, "sha512-ViRBkoZD9Rk0hGeMdd2GHGaOaZuH9mDmwsE5/Zo53Ftwcvh7h9VJc8lIt2wdgEwS4EW5lbtTX6vlE0idCLPOyA=="],
+
     "@types/estree": ["@types/estree@1.0.6", "", {}, "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw=="],
 
     "@types/fuzzy-search": ["@types/fuzzy-search@2.1.5", "", {}, "sha512-Yw8OsjhVKbKw83LMDOZ9RXc+N+um48DmZYMrz7QChpHkQuygsc5O40oCL7SfvWgpaaviCx2TbNXYUBwhMtBH5w=="],
 
     "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="],
 
+    "@types/luxon": ["@types/luxon@3.4.2", "", {}, "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA=="],
+
     "@types/node": ["@types/node@22.13.8", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-G3EfaZS+iOGYWLLRCEAXdWK9my08oHNZ+FHluRiggIYJPOXzhOiDgpVCUHaUvyIC5/fj7C/p637jdzC666AOKQ=="],
 
     "@types/resolve": ["@types/resolve@1.20.2", "", {}, "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q=="],
@@ -287,6 +295,8 @@
 
     "cookie": ["cookie@0.6.0", "", {}, "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw=="],
 
+    "cron": ["cron@4.1.0", "", { "dependencies": { "@types/luxon": "~3.4.0", "luxon": "~3.5.0" } }, "sha512-wmcuXr2qP0UZStYgwruG6jC2AYSO9n5VMm2t93hmcEXEjWY3S2bsXe3sfGUrTs/uQ1AvRCrZ0Pp9Q032L/V9tw=="],
+
     "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
 
     "cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="],
@@ -451,6 +461,8 @@
 
     "lodash.merge": ["lodash.merge@4.6.2", "", {}, "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="],
 
+    "luxon": ["luxon@3.5.0", "", {}, "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="],
+
     "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="],
 
     "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="],

package.json

@@ -10,6 +10,7 @@
     "@tailwindcss/forms": "^0.5.9",
     "@tailwindcss/typography": "^0.5.15",
     "@tailwindcss/vite": "^4.0.0",
+    "@types/cron": "^2.4.3",
     "@types/fuzzy-search": "^2.1.5",
     "bun-types": "^1.2.4",
     "daisyui": "^5.0.0",
@@ -39,5 +40,8 @@
     "format": "prettier --write .",
     "lint": "prettier --check . && eslint ."
   },
-  "type": "module"
+  "type": "module",
+  "dependencies": {
+    "cron": "^4.1.0"
+  }
 }

src/hooks.server.ts

@@ -0,0 +1,3 @@
+import { startTemperatureCronJob } from '$lib/server/cronJobs';
+
+startTemperatureCronJob();

src/lib/db/queries.ts

@@ -12,13 +12,25 @@ export const allCountries = db.query(`
 `);
 
 export const allCities = db.query("SELECT id, city as name, country_id, lat as latitude, lng as longitude FROM cities WHERE capital IN ('capital', 'admin') OR (population >= 100000 AND population != '')");
+export const allCitiesWithNoTemperatureToday = db.query(`
+    SELECT c.id, c.city as name, c.country_id, c.lat as latitude, c.lng as longitude
+    FROM cities c
+    LEFT JOIN meteo_data md ON c.id = md.city_id AND md.date = $date
+    WHERE (c.capital IN ('capital', 'admin') OR (c.population >= 100000 AND c.population != ''))
+    AND md.city_id IS NULL;
+`);
 
 export const cityTemperatureToday = db.query(`
-    SELECT c.id, c.city as name, md.min, md.max
+    SELECT c.id, c.city as name, md.min, md.max, md.date
     FROM cities as c
     JOIN meteo_data as md ON c.id = md.city_id
-    WHERE c.id = $1
-    AND md.date = $2
+    WHERE c.id = $id
+    AND md.date = $date
+`);
+
+export const insertCityTemperatureData = db.query(`
+    INSERT INTO meteo_data (city_id, min, max, date)
+    VALUES ($id, $min, $max, $date)
 `);
 
 // export const countryExtremeTemperatures = db.query(`

src/lib/server/cronJobs.ts

@@ -0,0 +1,22 @@
+import { CronJob } from 'cron';
+import { fetchAllCitiesTemperatures } from './meteoDataHandler';
+
+const today = new Date();
+const shortDate = today.toISOString().split('T')[0];
+const temperatureCronJob = new CronJob('0 0 * * *', () => {
+    console.log('Running daily temperature fetch for: ', shortDate);
+    fetchAllCitiesTemperatures(shortDate);
+}, () => {
+    console.log('Temperature cron job completed for: ', shortDate);
+}, true, 'UTC');
+
+// the date will be saved in string format "YYYY-MM-DD"
+export function startTemperatureCronJob() {
+    temperatureCronJob.start();
+    console.log('Temperature cron job scheduled');
+    
+    // Run immediately on startup if no data exists for today
+    if (today.getUTCHours() > 0) { // Only run if it's past midnight UTC
+        fetchAllCitiesTemperatures(shortDate);
+    }
+}

src/lib/server/meteoDataHandler.ts

@@ -0,0 +1,55 @@
+import { allCities, allCitiesWithNoTemperatureToday, cityTemperatureToday, insertCityTemperatureData } from "$lib/db/queries";
+import type { CityBasic, CityTemperature } from "$lib/types";
+import { getLocationTemperature } from "./meteoService";
+
+const COOLDOWN_TIME = 500;
+
+// TODO: Make sure if the job process is staled restart it and refetch the data
+export const fetchAllCitiesTemperatures = async (date: string) => {
+    try {
+
+        const cities = allCitiesWithNoTemperatureToday.all({ $date: date }) as CityBasic[];
+
+        console.log('Today date: ', date);
+        console.log('Total Cities to fetch: ', cities.length);
+
+        for (const city of cities) {
+            try {
+                const existingRecord = cityTemperatureToday.get({ $id: city.id, $date: date }) as CityTemperature | null;
+                if (!existingRecord) {
+                    
+                    const { data } = await getLocationTemperature(
+                        city.id,
+                        city.latitude.toString(),
+                        city.longitude.toString()
+                    );
+                    // Add cooldown delay to avoid API throttling
+                    if (data) {
+                        insertCityTemperatureData.run({
+                            $id: data.id,
+                            $min: data.min,
+                            $max: data.max,
+                            $date: data.date
+                        });
+                        console.log(`Temperature data stored for ${city.name}`);
+                    }
+                    await new Promise(resolve => setTimeout(resolve, COOLDOWN_TIME));
+                } else {
+                    console.log(`Temperature data already exists for ${city.name}`);
+                }
+
+
+            } catch (error) {
+                console.error(`Failed to fetch temperature for city ${city.name}:`, error);
+            }
+        }
+        return {
+            ok: true,
+        }
+    } catch (error) {
+        console.error('Failed to fetch cities:', error);
+        return {
+            ok: false
+        }
+    }
+}

src/lib/server/meteoService.ts

@@ -1,5 +1,6 @@
+import type { MeteoData, MeteoResponseType } from "$lib/types";
 
-export const getLocationTemperature = async (latitude: string, longitude: string) => {
+export const getLocationTemperature = async (id: number, latitude: string, longitude: string) => {
     const meteoEndpoint = "https://api.open-meteo.com/v1/forecast";
     //   https://api.open-meteo.com/v1/forecast?latitude=48.0356&longitude=27.8129&hourly=temperature_2m&daily=temperature_2m_max,temperature_2m_min&forecast_days=1&format=json&timeformat=unixtime
 
@@ -14,9 +15,16 @@ export const getLocationTemperature = async (latitude: string, longitude: string
     try {
         const response = await fetch(url);
         if (response.ok) {
-            const data = await response.json();
+            const data = await response.json() as MeteoResponseType;
             console.log('Fetched DATA from open meteo >>>>');
-            return { data };
+            return {
+                data: {
+                    id,
+                    max: data.daily.temperature_2m_max[0],
+                    min: data.daily.temperature_2m_min[0],
+                    date: data.daily.time[0],
+                } as MeteoData
+            };
         } else {
             return { data: null };
         }

src/lib/types.ts

@@ -13,9 +13,43 @@ export type CityBasic = {
   longitude: number;
 };
 
+export type CityTemperature = {
+  id: number;
+  name: string;
+  min: number;
+  max: number;
+  date: string;
+}
 export type ComboOption = {
   id: number;
   name: string;
   emoji?: string;
   country_id?: number;
 }
+
+export type MeteoResponseType = {
+  latitude: number;
+  longitude: number;
+  generationtime_ms: number;
+  utc_offset_seconds: number;
+  timezone: string;
+  timezone_abbreviation: string;
+  elevation: number;
+  daily_units: {
+    time: string;
+    temperature_2m_max: string;
+    temperature_2m_min: string;
+  },
+  daily: {
+    time: string[];
+    temperature_2m_max: number[];
+    temperature_2m_min: number[];
+  },
+}
+
+export type MeteoData = {
+  id: number;
+  max: number;
+  min: number;
+  date: string;
+}

src/routes/+page.server.ts

@@ -3,6 +3,7 @@ import type { CityBasic, CountryBasic } from '$lib/types';
 import { error, type Actions } from '@sveltejs/kit';
 import { getLocationTemperature } from '$lib/server/meteoService';
 import type { PageServerLoad } from './$types';
+import { fetchAllCitiesTemperatures } from '$lib/server/meteoDataHandler';
 
 const errorObject = {
   ok: false,
@@ -31,11 +32,15 @@ export const actions: Actions = {
   fetchTemperature: async ({ request }) => {
     try {
       const formData = await request.formData();
+      const id = formData.get('id') as string;
       const latitude = formData.get('latitude') as string;
       const longitude = formData.get('longitude') as string;
-			const response = await getLocationTemperature(latitude, longitude);
-			console.log('RESPONSE >>>>', response);
+      // const response = await fetchAllCitiesTemperatures(new Date().toISOString().split('T')[0]);
+      // console.log('RESPONSE >>>>', response);
+      const response = await getLocationTemperature(Number(id), latitude, longitude);
       if (response.data) {
+        console.log(response.data);
+
         return {
           ok: true,
           data: response.data,
@@ -44,6 +49,11 @@ export const actions: Actions = {
       } else {
         return errorObject;
       }
+      // return {
+      // 	ok: true,
+      // 	data: response,
+      // 	error: null
+      // }
     } catch (error) {
       console.log(error);
       return errorObject;