gitea_admin/hot-cold-cities
1
0
Fork 0
mirror of https://github.com/gevera/hot-cold-cities.git synced 2025-12-06 07:08:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added CRON job to fetch data

Browse Source
This commit is contained in:
Denis Donici 2025-03-06 16:54:14 +02:00
parent bb5a94b633
commit b14a55de71
10 changed files with 205 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
bun.lock
View File

@ -3,6 +3,9 @@
"workspaces": {
"": {
"name": "hc-app",
"dependencies": {
"cron": "^4.1.0",
},
"devDependencies": {
"@eslint/compat": "^1.2.5",
"@eslint/js": "^9.18.0",
@ -12,6 +15,7 @@
"@tailwindcss/forms": "^0.5.9",
"@tailwindcss/typography": "^0.5.15",
"@tailwindcss/vite": "^4.0.0",
"@types/cron": "^2.4.3",
"@types/fuzzy-search": "^2.1.5",
"bun-types": "^1.2.4",
"daisyui": "^5.0.0",
@ -217,12 +221,16 @@
"@types/cookie": ["@types/cookie@0.6.0", "", {}, "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA=="],
"@types/cron": ["@types/cron@2.4.3", "", { "dependencies": { "cron": "*" } }, "sha512-ViRBkoZD9Rk0hGeMdd2GHGaOaZuH9mDmwsE5/Zo53Ftwcvh7h9VJc8lIt2wdgEwS4EW5lbtTX6vlE0idCLPOyA=="],
"@types/estree": ["@types/estree@1.0.6", "", {}, "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw=="],
"@types/fuzzy-search": ["@types/fuzzy-search@2.1.5", "", {}, "sha512-Yw8OsjhVKbKw83LMDOZ9RXc+N+um48DmZYMrz7QChpHkQuygsc5O40oCL7SfvWgpaaviCx2TbNXYUBwhMtBH5w=="],
"@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="],
"@types/luxon": ["@types/luxon@3.4.2", "", {}, "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA=="],
"@types/node": ["@types/node@22.13.8", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-G3EfaZS+iOGYWLLRCEAXdWK9my08oHNZ+FHluRiggIYJPOXzhOiDgpVCUHaUvyIC5/fj7C/p637jdzC666AOKQ=="],
"@types/resolve": ["@types/resolve@1.20.2", "", {}, "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q=="],
@ -287,6 +295,8 @@
"cookie": ["cookie@0.6.0", "", {}, "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw=="],
"cron": ["cron@4.1.0", "", { "dependencies": { "@types/luxon": "~3.4.0", "luxon": "~3.5.0" } }, "sha512-wmcuXr2qP0UZStYgwruG6jC2AYSO9n5VMm2t93hmcEXEjWY3S2bsXe3sfGUrTs/uQ1AvRCrZ0Pp9Q032L/V9tw=="],
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
"cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="],
@ -451,6 +461,8 @@
"lodash.merge": ["lodash.merge@4.6.2", "", {}, "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="],
"luxon": ["luxon@3.5.0", "", {}, "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="],
"magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="],
"merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="],

BIN
database/data.db
View File

Binary file not shown.

6
package.json
View File

@ -10,6 +10,7 @@
"@tailwindcss/forms": "^0.5.9",
"@tailwindcss/typography": "^0.5.15",
"@tailwindcss/vite": "^4.0.0",
"@types/cron": "^2.4.3",
"@types/fuzzy-search": "^2.1.5",
"bun-types": "^1.2.4",
"daisyui": "^5.0.0",
@ -39,5 +40,8 @@
"format": "prettier --write .",
"lint": "prettier --check . && eslint ."
},
"type": "module"
"type": "module",
"dependencies": {
"cron": "^4.1.0"
}
}

3
src/hooks.server.ts Normal file
View File

@ -0,0 +1,3 @@
import { startTemperatureCronJob } from '$lib/server/cronJobs';
startTemperatureCronJob();

18
src/lib/db/queries.ts
View File

@ -12,13 +12,25 @@ export const allCountries = db.query(`
`);
export const allCities = db.query("SELECT id, city as name, country_id, lat as latitude, lng as longitude FROM cities WHERE capital IN ('capital', 'admin') OR (population >= 100000 AND population != '')");
export const allCitiesWithNoTemperatureToday = db.query(`
SELECT c.id, c.city as name, c.country_id, c.lat as latitude, c.lng as longitude
FROM cities c
LEFT JOIN meteo_data md ON c.id = md.city_id AND md.date = $date
WHERE (c.capital IN ('capital', 'admin') OR (c.population >= 100000 AND c.population != ''))
AND md.city_id IS NULL;
`);
export const cityTemperatureToday = db.query(`
SELECT c.id, c.city as name, md.min, md.max
SELECT c.id, c.city as name, md.min, md.max, md.date
FROM cities as c
JOIN meteo_data as md ON c.id = md.city_id
WHERE c.id = $1
AND md.date = $2
WHERE c.id = $id
AND md.date = $date
`);
export const insertCityTemperatureData = db.query(`
INSERT INTO meteo_data (city_id, min, max, date)
VALUES ($id, $min, $max, $date)
`);
// export const countryExtremeTemperatures = db.query(`

22
src/lib/server/cronJobs.ts Normal file
View File

@ -0,0 +1,22 @@
import { CronJob } from 'cron';
import { fetchAllCitiesTemperatures } from './meteoDataHandler';
const today = new Date();
const shortDate = today.toISOString().split('T')[0];
const temperatureCronJob = new CronJob('0 0 * * *', () => {
console.log('Running daily temperature fetch for: ', shortDate);
fetchAllCitiesTemperatures(shortDate);
}, () => {
console.log('Temperature cron job completed for: ', shortDate);
}, true, 'UTC');
// the date will be saved in string format "YYYY-MM-DD"
export function startTemperatureCronJob() {
temperatureCronJob.start();
console.log('Temperature cron job scheduled');
// Run immediately on startup if no data exists for today
if (today.getUTCHours() > 0) { // Only run if it's past midnight UTC
fetchAllCitiesTemperatures(shortDate);
}
}

55
src/lib/server/meteoDataHandler.ts Normal file
View File

@ -0,0 +1,55 @@
import { allCities, allCitiesWithNoTemperatureToday, cityTemperatureToday, insertCityTemperatureData } from "$lib/db/queries";
import type { CityBasic, CityTemperature } from "$lib/types";
import { getLocationTemperature } from "./meteoService";
const COOLDOWN_TIME = 500;
// TODO: Make sure if the job process is staled restart it and refetch the data
export const fetchAllCitiesTemperatures = async (date: string) => {
try {
const cities = allCitiesWithNoTemperatureToday.all({ $date: date }) as CityBasic[];
console.log('Today date: ', date);
console.log('Total Cities to fetch: ', cities.length);
for (const city of cities) {
try {
const existingRecord = cityTemperatureToday.get({ $id: city.id, $date: date }) as CityTemperature | null;
if (!existingRecord) {
const { data } = await getLocationTemperature(
city.id,
city.latitude.toString(),
city.longitude.toString()
);
// Add cooldown delay to avoid API throttling
if (data) {
insertCityTemperatureData.run({
$id: data.id,
$min: data.min,
$max: data.max,
$date: data.date
});
console.log(`Temperature data stored for ${city.name}`);
}
await new Promise(resolve => setTimeout(resolve, COOLDOWN_TIME));
} else {
console.log(`Temperature data already exists for ${city.name}`);
}
} catch (error) {
console.error(`Failed to fetch temperature for city ${city.name}:`, error);
}
}
return {
ok: true,
}
} catch (error) {
console.error('Failed to fetch cities:', error);
return {
ok: false
}
}
}

14
src/lib/server/meteoService.ts
View File

@ -1,5 +1,6 @@
import type { MeteoData, MeteoResponseType } from "$lib/types";
export const getLocationTemperature = async (latitude: string, longitude: string) => {
export const getLocationTemperature = async (id: number, latitude: string, longitude: string) => {
const meteoEndpoint = "https://api.open-meteo.com/v1/forecast";
// https://api.open-meteo.com/v1/forecast?latitude=48.0356&longitude=27.8129&hourly=temperature_2m&daily=temperature_2m_max,temperature_2m_min&forecast_days=1&format=json&timeformat=unixtime
@ -14,9 +15,16 @@ export const getLocationTemperature = async (latitude: string, longitude: string
try {
const response = await fetch(url);
if (response.ok) {
const data = await response.json();
const data = await response.json() as MeteoResponseType;
console.log('Fetched DATA from open meteo >>>>');
return { data };
return {
data: {
id,
max: data.daily.temperature_2m_max[0],
min: data.daily.temperature_2m_min[0],
date: data.daily.time[0],
} as MeteoData
};
} else {
return { data: null };
}

34
src/lib/types.ts
View File

@ -13,9 +13,43 @@ export type CityBasic = {
longitude: number;
};
export type CityTemperature = {
id: number;
name: string;
min: number;
max: number;
date: string;
}
export type ComboOption = {
id: number;
name: string;
emoji?: string;
country_id?: number;
}
export type MeteoResponseType = {
latitude: number;
longitude: number;
generationtime_ms: number;
utc_offset_seconds: number;
timezone: string;
timezone_abbreviation: string;
elevation: number;
daily_units: {
time: string;
temperature_2m_max: string;
temperature_2m_min: string;
},
daily: {
time: string[];
temperature_2m_max: number[];
temperature_2m_min: number[];
},
}
export type MeteoData = {
id: number;
max: number;
min: number;
date: string;
}

86
src/routes/+page.server.ts
View File

@ -3,51 +3,61 @@ import type { CityBasic, CountryBasic } from '$lib/types';
import { error, type Actions } from '@sveltejs/kit';
import { getLocationTemperature } from '$lib/server/meteoService';
import type { PageServerLoad } from './$types';
import { fetchAllCitiesTemperatures } from '$lib/server/meteoDataHandler';
const errorObject = {
ok: false,
data: null,
error: 'Failed to fetch temperature'
ok: false,
data: null,
error: 'Failed to fetch temperature'
}
export const load: PageServerLoad = () => {
try {
const countries = allCountries.all() as CountryBasic[];
const cities = allCities.all() as CityBasic[];
return {
countries,
cities
};
} catch (error) {
console.log(error);
return {
countries: [],
cities: []
};
}
try {
const countries = allCountries.all() as CountryBasic[];
const cities = allCities.all() as CityBasic[];
return {
countries,
cities
};
} catch (error) {
console.log(error);
return {
countries: [],
cities: []
};
}
};
export const actions: Actions = {
fetchTemperature: async ({ request }) => {
try {
const formData = await request.formData();
const latitude = formData.get('latitude') as string;
const longitude = formData.get('longitude') as string;
const response = await getLocationTemperature(latitude, longitude);
console.log('RESPONSE >>>>', response);
if (response.data) {
return {
ok: true,
data: response.data,
error: null
};
} else {
return errorObject;
}
} catch (error) {
console.log(error);
return errorObject;
}
fetchTemperature: async ({ request }) => {
try {
const formData = await request.formData();
const id = formData.get('id') as string;
const latitude = formData.get('latitude') as string;
const longitude = formData.get('longitude') as string;
// const response = await fetchAllCitiesTemperatures(new Date().toISOString().split('T')[0]);
// console.log('RESPONSE >>>>', response);
const response = await getLocationTemperature(Number(id), latitude, longitude);
if (response.data) {
console.log(response.data);
}
return {
ok: true,
data: response.data,
error: null
};
} else {
return errorObject;
}
// return {
// ok: true,
// data: response,
// error: null
// }
} catch (error) {
console.log(error);
return errorObject;
}
}
}