diff --git a/bun.lock b/bun.lock index 539f339..7985cd0 100644 --- a/bun.lock +++ b/bun.lock @@ -3,6 +3,9 @@ "workspaces": { "": { "name": "hc-app", + "dependencies": { + "cron": "^4.1.0", + }, "devDependencies": { "@eslint/compat": "^1.2.5", "@eslint/js": "^9.18.0", @@ -12,6 +15,7 @@ "@tailwindcss/forms": "^0.5.9", "@tailwindcss/typography": "^0.5.15", "@tailwindcss/vite": "^4.0.0", + "@types/cron": "^2.4.3", "@types/fuzzy-search": "^2.1.5", "bun-types": "^1.2.4", "daisyui": "^5.0.0", @@ -217,12 +221,16 @@ "@types/cookie": ["@types/cookie@0.6.0", "", {}, "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA=="], + "@types/cron": ["@types/cron@2.4.3", "", { "dependencies": { "cron": "*" } }, "sha512-ViRBkoZD9Rk0hGeMdd2GHGaOaZuH9mDmwsE5/Zo53Ftwcvh7h9VJc8lIt2wdgEwS4EW5lbtTX6vlE0idCLPOyA=="], + "@types/estree": ["@types/estree@1.0.6", "", {}, "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw=="], "@types/fuzzy-search": ["@types/fuzzy-search@2.1.5", "", {}, "sha512-Yw8OsjhVKbKw83LMDOZ9RXc+N+um48DmZYMrz7QChpHkQuygsc5O40oCL7SfvWgpaaviCx2TbNXYUBwhMtBH5w=="], "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], + "@types/luxon": ["@types/luxon@3.4.2", "", {}, "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA=="], + "@types/node": ["@types/node@22.13.8", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-G3EfaZS+iOGYWLLRCEAXdWK9my08oHNZ+FHluRiggIYJPOXzhOiDgpVCUHaUvyIC5/fj7C/p637jdzC666AOKQ=="], "@types/resolve": ["@types/resolve@1.20.2", "", {}, "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q=="], @@ -287,6 +295,8 @@ "cookie": ["cookie@0.6.0", "", {}, "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw=="], + "cron": ["cron@4.1.0", "", { "dependencies": { "@types/luxon": "~3.4.0", "luxon": "~3.5.0" } }, "sha512-wmcuXr2qP0UZStYgwruG6jC2AYSO9n5VMm2t93hmcEXEjWY3S2bsXe3sfGUrTs/uQ1AvRCrZ0Pp9Q032L/V9tw=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="], @@ -451,6 +461,8 @@ "lodash.merge": ["lodash.merge@4.6.2", "", {}, "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="], + "luxon": ["luxon@3.5.0", "", {}, "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="], + "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="], "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], diff --git a/database/data.db b/database/data.db index bea363a..8814b2a 100644 Binary files a/database/data.db and b/database/data.db differ diff --git a/package.json b/package.json index 137d8f8..fc373ba 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "@tailwindcss/forms": "^0.5.9", "@tailwindcss/typography": "^0.5.15", "@tailwindcss/vite": "^4.0.0", + "@types/cron": "^2.4.3", "@types/fuzzy-search": "^2.1.5", "bun-types": "^1.2.4", "daisyui": "^5.0.0", @@ -39,5 +40,8 @@ "format": "prettier --write .", "lint": "prettier --check . && eslint ." }, - "type": "module" + "type": "module", + "dependencies": { + "cron": "^4.1.0" + } } diff --git a/src/hooks.server.ts b/src/hooks.server.ts new file mode 100644 index 0000000..96dbfb3 --- /dev/null +++ b/src/hooks.server.ts @@ -0,0 +1,3 @@ +import { startTemperatureCronJob } from '$lib/server/cronJobs'; + +startTemperatureCronJob(); diff --git a/src/lib/db/queries.ts b/src/lib/db/queries.ts index 2a95564..b66cfa1 100644 --- a/src/lib/db/queries.ts +++ b/src/lib/db/queries.ts @@ -12,13 +12,25 @@ export const allCountries = db.query(` `); export const allCities = db.query("SELECT id, city as name, country_id, lat as latitude, lng as longitude FROM cities WHERE capital IN ('capital', 'admin') OR (population >= 100000 AND population != '')"); +export const allCitiesWithNoTemperatureToday = db.query(` + SELECT c.id, c.city as name, c.country_id, c.lat as latitude, c.lng as longitude + FROM cities c + LEFT JOIN meteo_data md ON c.id = md.city_id AND md.date = $date + WHERE (c.capital IN ('capital', 'admin') OR (c.population >= 100000 AND c.population != '')) + AND md.city_id IS NULL; +`); export const cityTemperatureToday = db.query(` - SELECT c.id, c.city as name, md.min, md.max + SELECT c.id, c.city as name, md.min, md.max, md.date FROM cities as c JOIN meteo_data as md ON c.id = md.city_id - WHERE c.id = $1 - AND md.date = $2 + WHERE c.id = $id + AND md.date = $date +`); + +export const insertCityTemperatureData = db.query(` + INSERT INTO meteo_data (city_id, min, max, date) + VALUES ($id, $min, $max, $date) `); // export const countryExtremeTemperatures = db.query(` diff --git a/src/lib/server/cronJobs.ts b/src/lib/server/cronJobs.ts new file mode 100644 index 0000000..a9f0cf1 --- /dev/null +++ b/src/lib/server/cronJobs.ts @@ -0,0 +1,22 @@ +import { CronJob } from 'cron'; +import { fetchAllCitiesTemperatures } from './meteoDataHandler'; + +const today = new Date(); +const shortDate = today.toISOString().split('T')[0]; +const temperatureCronJob = new CronJob('0 0 * * *', () => { + console.log('Running daily temperature fetch for: ', shortDate); + fetchAllCitiesTemperatures(shortDate); +}, () => { + console.log('Temperature cron job completed for: ', shortDate); +}, true, 'UTC'); + +// the date will be saved in string format "YYYY-MM-DD" +export function startTemperatureCronJob() { + temperatureCronJob.start(); + console.log('Temperature cron job scheduled'); + + // Run immediately on startup if no data exists for today + if (today.getUTCHours() > 0) { // Only run if it's past midnight UTC + fetchAllCitiesTemperatures(shortDate); + } +} \ No newline at end of file diff --git a/src/lib/server/meteoDataHandler.ts b/src/lib/server/meteoDataHandler.ts new file mode 100644 index 0000000..a4762e1 --- /dev/null +++ b/src/lib/server/meteoDataHandler.ts @@ -0,0 +1,55 @@ +import { allCities, allCitiesWithNoTemperatureToday, cityTemperatureToday, insertCityTemperatureData } from "$lib/db/queries"; +import type { CityBasic, CityTemperature } from "$lib/types"; +import { getLocationTemperature } from "./meteoService"; + +const COOLDOWN_TIME = 500; + +// TODO: Make sure if the job process is staled restart it and refetch the data +export const fetchAllCitiesTemperatures = async (date: string) => { + try { + + const cities = allCitiesWithNoTemperatureToday.all({ $date: date }) as CityBasic[]; + + console.log('Today date: ', date); + console.log('Total Cities to fetch: ', cities.length); + + for (const city of cities) { + try { + const existingRecord = cityTemperatureToday.get({ $id: city.id, $date: date }) as CityTemperature | null; + if (!existingRecord) { + + const { data } = await getLocationTemperature( + city.id, + city.latitude.toString(), + city.longitude.toString() + ); + // Add cooldown delay to avoid API throttling + if (data) { + insertCityTemperatureData.run({ + $id: data.id, + $min: data.min, + $max: data.max, + $date: data.date + }); + console.log(`Temperature data stored for ${city.name}`); + } + await new Promise(resolve => setTimeout(resolve, COOLDOWN_TIME)); + } else { + console.log(`Temperature data already exists for ${city.name}`); + } + + + } catch (error) { + console.error(`Failed to fetch temperature for city ${city.name}:`, error); + } + } + return { + ok: true, + } + } catch (error) { + console.error('Failed to fetch cities:', error); + return { + ok: false + } + } +} \ No newline at end of file diff --git a/src/lib/server/meteoService.ts b/src/lib/server/meteoService.ts index 05f78cc..8252535 100644 --- a/src/lib/server/meteoService.ts +++ b/src/lib/server/meteoService.ts @@ -1,5 +1,6 @@ +import type { MeteoData, MeteoResponseType } from "$lib/types"; -export const getLocationTemperature = async (latitude: string, longitude: string) => { +export const getLocationTemperature = async (id: number, latitude: string, longitude: string) => { const meteoEndpoint = "https://api.open-meteo.com/v1/forecast"; // https://api.open-meteo.com/v1/forecast?latitude=48.0356&longitude=27.8129&hourly=temperature_2m&daily=temperature_2m_max,temperature_2m_min&forecast_days=1&format=json&timeformat=unixtime @@ -14,9 +15,16 @@ export const getLocationTemperature = async (latitude: string, longitude: string try { const response = await fetch(url); if (response.ok) { - const data = await response.json(); + const data = await response.json() as MeteoResponseType; console.log('Fetched DATA from open meteo >>>>'); - return { data }; + return { + data: { + id, + max: data.daily.temperature_2m_max[0], + min: data.daily.temperature_2m_min[0], + date: data.daily.time[0], + } as MeteoData + }; } else { return { data: null }; } diff --git a/src/lib/types.ts b/src/lib/types.ts index 621daa0..1e192c7 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -13,9 +13,43 @@ export type CityBasic = { longitude: number; }; +export type CityTemperature = { + id: number; + name: string; + min: number; + max: number; + date: string; +} export type ComboOption = { id: number; name: string; emoji?: string; country_id?: number; } + +export type MeteoResponseType = { + latitude: number; + longitude: number; + generationtime_ms: number; + utc_offset_seconds: number; + timezone: string; + timezone_abbreviation: string; + elevation: number; + daily_units: { + time: string; + temperature_2m_max: string; + temperature_2m_min: string; + }, + daily: { + time: string[]; + temperature_2m_max: number[]; + temperature_2m_min: number[]; + }, +} + +export type MeteoData = { + id: number; + max: number; + min: number; + date: string; +} diff --git a/src/routes/+page.server.ts b/src/routes/+page.server.ts index 743f889..fb64b78 100644 --- a/src/routes/+page.server.ts +++ b/src/routes/+page.server.ts @@ -3,51 +3,61 @@ import type { CityBasic, CountryBasic } from '$lib/types'; import { error, type Actions } from '@sveltejs/kit'; import { getLocationTemperature } from '$lib/server/meteoService'; import type { PageServerLoad } from './$types'; +import { fetchAllCitiesTemperatures } from '$lib/server/meteoDataHandler'; const errorObject = { - ok: false, - data: null, - error: 'Failed to fetch temperature' + ok: false, + data: null, + error: 'Failed to fetch temperature' } export const load: PageServerLoad = () => { - try { - const countries = allCountries.all() as CountryBasic[]; - const cities = allCities.all() as CityBasic[]; - return { - countries, - cities - }; - } catch (error) { - console.log(error); - return { - countries: [], - cities: [] - }; - } + try { + const countries = allCountries.all() as CountryBasic[]; + const cities = allCities.all() as CityBasic[]; + return { + countries, + cities + }; + } catch (error) { + console.log(error); + return { + countries: [], + cities: [] + }; + } }; export const actions: Actions = { - fetchTemperature: async ({ request }) => { - try { - const formData = await request.formData(); - const latitude = formData.get('latitude') as string; - const longitude = formData.get('longitude') as string; - const response = await getLocationTemperature(latitude, longitude); - console.log('RESPONSE >>>>', response); - if (response.data) { - return { - ok: true, - data: response.data, - error: null - }; - } else { - return errorObject; - } - } catch (error) { - console.log(error); - return errorObject; - } + fetchTemperature: async ({ request }) => { + try { + const formData = await request.formData(); + const id = formData.get('id') as string; + const latitude = formData.get('latitude') as string; + const longitude = formData.get('longitude') as string; + // const response = await fetchAllCitiesTemperatures(new Date().toISOString().split('T')[0]); + // console.log('RESPONSE >>>>', response); + const response = await getLocationTemperature(Number(id), latitude, longitude); + if (response.data) { + console.log(response.data); - } + return { + ok: true, + data: response.data, + error: null + }; + } else { + return errorObject; + } + // return { + // ok: true, + // data: response, + // error: null + // } + } catch (error) { + console.log(error); + return errorObject; + } + + } }