Compare commits

..

No commits in common. "ace56570573adbda671d660ba0e854a4ab038082" and "d5f63107b9dc41273bdca9535cdd7a21a7405023" have entirely different histories.

22 changed files with 96 additions and 373 deletions

View file

@ -1,11 +1,6 @@
module.exports = async function ({ services: { authTokenHandler } }) { module.exports = async function ({ services: { authTokenHandler } }) {
async function doAuthLoginToken(req) { async function doAuthLoginToken(req) {
const { const { authTokenJwt, phoneModel = null, deviceUuid = null } = req.body
authTokenJwt,
phoneModel = null,
phoneOS = null,
deviceUuid = null,
} = req.body
// Validate the auth token JWT and extract the auth token // Validate the auth token JWT and extract the auth token
const authToken = authTokenHandler.decodeAuthToken(authTokenJwt) const authToken = authTokenHandler.decodeAuthToken(authTokenJwt)
@ -15,7 +10,6 @@ module.exports = async function ({ services: { authTokenHandler } }) {
await authTokenHandler.getOrCreateUserSession( await authTokenHandler.getOrCreateUserSession(
authToken, authToken,
phoneModel, phoneModel,
phoneOS,
deviceUuid deviceUuid
) )

View file

@ -11,8 +11,6 @@ requestBody:
format: apiJwt format: apiJwt
phoneModel: phoneModel:
type: string type: string
phoneOS:
type: string
deviceUuid: deviceUuid:
type: string type: string
format: uuid format: uuid

View file

@ -22,7 +22,6 @@ module.exports = ({ services }) => {
async function getOrCreateUserSession( async function getOrCreateUserSession(
authToken, authToken,
phoneModel = null, phoneModel = null,
phoneOS = null,
deviceUuid = null deviceUuid = null
) { ) {
let userId let userId
@ -55,8 +54,8 @@ module.exports = ({ services }) => {
id id
` `
;[{ id: deviceId }] = await sql` ;[{ id: deviceId }] = await sql`
INSERT INTO "device" ("user_id", "phone_model", "phone_os", "uuid") INSERT INTO "device" ("user_id", "phone_model", "uuid")
VALUES (${userId}, ${phoneModel}, ${phoneOS}, ${deviceUuid}) VALUES (${userId}, ${phoneModel}, ${deviceUuid})
RETURNING RETURNING
id id
` `
@ -105,8 +104,8 @@ module.exports = ({ services }) => {
if (!deviceId) { if (!deviceId) {
// Only create new device if UUID doesn't exist // Only create new device if UUID doesn't exist
;[{ id: deviceId }] = await sql` ;[{ id: deviceId }] = await sql`
INSERT INTO "device" ("user_id", "phone_model", "phone_os", "uuid") INSERT INTO "device" ("user_id", "phone_model", "uuid")
VALUES (${userId}, ${phoneModel}, ${phoneOS}, ${deviceUuid}) VALUES (${userId}, ${phoneModel}, ${deviceUuid})
RETURNING RETURNING
id id
` `

View file

@ -3,22 +3,10 @@ table:
schema: public schema: public
configuration: configuration:
column_config: column_config:
acknowledged_around_count:
custom_name: acknowledgedAroundCount
acknowledged_connect_count:
custom_name: acknowledgedConnectCount
acknowledged_relative_count:
custom_name: acknowledgedRelativeCount
alert_id: alert_id:
custom_name: alertId custom_name: alertId
alert_tag: alert_tag:
custom_name: alertTag custom_name: alertTag
alerting_around_count:
custom_name: alertingAroundCount
alerting_connect_count:
custom_name: alertingConnectCount
alerting_relative_count:
custom_name: alertingRelativeCount
altitude_accuracy: altitude_accuracy:
custom_name: altitudeAccuracy custom_name: altitudeAccuracy
archive_created_at: archive_created_at:
@ -33,20 +21,8 @@ configuration:
custom_name: createdAt custom_name: createdAt
device_id: device_id:
custom_name: deviceId custom_name: deviceId
emergency_calling_notification_sent:
custom_name: emergencyCallingNotificationSent
follow_location:
custom_name: followLocation
initial_location:
custom_name: initialLocation
keep_open_at: keep_open_at:
custom_name: keepOpenAt custom_name: keepOpenAt
last_address:
custom_name: lastAddress
last_nearest_place:
custom_name: lastNearestPlace
last_what3words:
custom_name: lastWhat3Words
nearest_place: nearest_place:
custom_name: nearestPlace custom_name: nearestPlace
notified_count: notified_count:
@ -68,14 +44,8 @@ configuration:
what3words: what3words:
custom_name: what3Words custom_name: what3Words
custom_column_names: custom_column_names:
acknowledged_around_count: acknowledgedAroundCount
acknowledged_connect_count: acknowledgedConnectCount
acknowledged_relative_count: acknowledgedRelativeCount
alert_id: alertId alert_id: alertId
alert_tag: alertTag alert_tag: alertTag
alerting_around_count: alertingAroundCount
alerting_connect_count: alertingConnectCount
alerting_relative_count: alertingRelativeCount
altitude_accuracy: altitudeAccuracy altitude_accuracy: altitudeAccuracy
archive_created_at: archiveCreatedAt archive_created_at: archiveCreatedAt
call_emergency: callEmergency call_emergency: callEmergency
@ -83,13 +53,7 @@ configuration:
closed_by: closedBy closed_by: closedBy
created_at: createdAt created_at: createdAt
device_id: deviceId device_id: deviceId
emergency_calling_notification_sent: emergencyCallingNotificationSent
follow_location: followLocation
initial_location: initialLocation
keep_open_at: keepOpenAt keep_open_at: keepOpenAt
last_address: lastAddress
last_nearest_place: lastNearestPlace
last_what3words: lastWhat3Words
nearest_place: nearestPlace nearest_place: nearestPlace
notified_count: notifiedCount notified_count: notifiedCount
notify_around: notifyAround notify_around: notifyAround

View file

@ -15,8 +15,6 @@ configuration:
custom_name: notificationAlertLevel custom_name: notificationAlertLevel
phone_model: phone_model:
custom_name: phoneModel custom_name: phoneModel
phone_os:
custom_name: phoneOs
preferred_emergency_call: preferred_emergency_call:
custom_name: preferredEmergencyCall custom_name: preferredEmergencyCall
radius_all: radius_all:
@ -34,7 +32,6 @@ configuration:
follow_location: followLocation follow_location: followLocation
notification_alert_level: notificationAlertLevel notification_alert_level: notificationAlertLevel
phone_model: phoneModel phone_model: phoneModel
phone_os: phoneOs
preferred_emergency_call: preferredEmergencyCall preferred_emergency_call: preferredEmergencyCall
radius_all: radiusAll radius_all: radiusAll
radius_reach: radiusReach radius_reach: radiusReach
@ -62,7 +59,6 @@ insert_permissions:
user_id: X-Hasura-User-Id user_id: X-Hasura-User-Id
columns: columns:
- phone_model - phone_model
- phone_os
- uuid - uuid
select_permissions: select_permissions:
- role: owner - role: owner
@ -77,7 +73,6 @@ select_permissions:
- location - location
- notification_alert_level - notification_alert_level
- phone_model - phone_model
- phone_os
- preferred_emergency_call - preferred_emergency_call
- radius_all - radius_all
- radius_reach - radius_reach

View file

@ -1,19 +0,0 @@
table:
name: enum_phone_os
schema: public
is_enum: true
configuration:
column_config: {}
custom_column_names: {}
custom_root_fields:
delete: deleteManyEnumPhoneOs
delete_by_pk: deleteOneEnumPhoneOs
insert: insertManyEnumPhoneOs
insert_one: insertOneEnumPhoneOs
select: selectManyEnumPhoneOs
select_aggregate: selectAggEnumPhoneOs
select_by_pk: selectOneEnumPhoneOs
select_stream: selectStreamEnumPhoneOs
update: updateManyEnumPhoneOs
update_by_pk: updateOneEnumPhoneOs
update_many: updateBatchEnumPhoneOs

View file

@ -19,7 +19,6 @@
- "!include public_enum_content_type.yaml" - "!include public_enum_content_type.yaml"
- "!include public_enum_emergency_call.yaml" - "!include public_enum_emergency_call.yaml"
- "!include public_enum_notification_type.yaml" - "!include public_enum_notification_type.yaml"
- "!include public_enum_phone_os.yaml"
- "!include public_enum_user_login_request_type.yaml" - "!include public_enum_user_login_request_type.yaml"
- "!include public_enum_user_role.yaml" - "!include public_enum_user_role.yaml"
- "!include public_external_public_config.yaml" - "!include public_external_public_config.yaml"

View file

@ -1,4 +0,0 @@
-- Could not auto-generate a down migration.
-- Please write an appropriate down migration for the SQL below:
-- alter table "public"."device" add column "phone_os" text
-- null;

View file

@ -1,2 +0,0 @@
alter table "public"."device" add column "phone_os" text
null;

View file

@ -1 +0,0 @@
DROP TABLE "public"."enum_phone_os";

View file

@ -1 +0,0 @@
CREATE TABLE "public"."enum_phone_os" ("value" text NOT NULL, PRIMARY KEY ("value") );

View file

@ -1 +0,0 @@
DELETE FROM "public"."enum_phone_os" WHERE "value" = 'ios';

View file

@ -1 +0,0 @@
INSERT INTO "public"."enum_phone_os"("value") VALUES (E'ios');

View file

@ -1 +0,0 @@
DELETE FROM "public"."enum_phone_os" WHERE "value" = 'android';

View file

@ -1 +0,0 @@
INSERT INTO "public"."enum_phone_os"("value") VALUES (E'android');

View file

@ -1 +0,0 @@
alter table "public"."device" drop constraint "device_phone_os_fkey";

View file

@ -1,5 +0,0 @@
alter table "public"."device"
add constraint "device_phone_os_fkey"
foreign key ("phone_os")
references "public"."enum_phone_os"
("value") on update restrict on delete restrict;

View file

@ -1,120 +0,0 @@
const { ctx } = require("@modjo/core")
const { taskCtx } = require("@modjo/microservice-worker/ctx")
const pushNotification = require("~/services/push-notification")
function createIosGeolocationHeartbeatSyncNotification() {
return {
data: {
action: "geolocation-heartbeat-sync",
},
// Silent push notification
notification: {
silent: true,
},
}
}
module.exports = async function () {
return Object.assign(
async function iosGeolocationHeartbeatSync(params) {
const logger = taskCtx.require("logger")
const sql = ctx.require("postgres")
const redisCold = ctx.require("keydbColdGeodata")
const { deviceId } = params
try {
// Get the device information including phone_os and fcm_token
const deviceResult = await sql`
SELECT
"user_id" as "userId",
"phone_os" as "phoneOs",
"fcm_token" as "fcmToken"
FROM
"device"
WHERE
id = ${deviceId}
`
if (!deviceResult || deviceResult.length === 0) {
logger.warn(
{ deviceId },
"No device found when sending iOS geolocation heartbeat sync"
)
return
}
const { userId, phoneOs, fcmToken } = deviceResult[0]
// Only proceed if device is iOS
if (phoneOs !== "ios") {
logger.debug(
{ deviceId, phoneOs },
"Skipping iOS heartbeat sync - device is not iOS"
)
return
}
if (!fcmToken) {
logger.warn(
{ deviceId, userId },
"No FCM token found for iOS device when sending heartbeat sync"
)
return
}
// Check if we've already sent a heartbeat sync for this device in the last 24h
const heartbeatSentKey = `ios_heartbeat_sent:device:${deviceId}`
const alreadySent = await redisCold.exists(heartbeatSentKey)
if (alreadySent) {
logger.debug(
{ deviceId, userId },
"iOS heartbeat sync already sent for this device in the last 24h"
)
return
}
// Create silent notification config
const notificationConfig =
createIosGeolocationHeartbeatSyncNotification()
// Send silent push notification
logger.info(
{ deviceId, userId, notificationConfig },
"Sending iOS silent push for geolocation heartbeat sync"
)
const { success } = await pushNotification({
fcmToken,
deviceId,
notification: notificationConfig.notification,
data: notificationConfig.data,
})
if (!success) {
throw new Error(
"Unable to send iOS geolocation heartbeat sync notification"
)
}
// Mark as sent with 24h expiry to prevent duplicate sends
await redisCold.set(heartbeatSentKey, "1", "EX", 24 * 60 * 60)
logger.info(
{ deviceId, userId },
"Successfully sent iOS geolocation heartbeat sync notification"
)
} catch (error) {
logger.error(
{ deviceId, error },
"Error sending iOS geolocation heartbeat sync notification"
)
}
},
{
dedupOptions: { enabled: true },
}
)
}

View file

@ -48,9 +48,7 @@ function deriveNotificationConfig({
android = {}, android = {},
apns = {}, apns = {},
uid, uid,
silent = false,
}) { }) {
const isVisible = !silent
const notification = { const notification = {
// https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.notification.md#notification_interface // https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.notification.md#notification_interface
title, title,
@ -76,14 +74,7 @@ function deriveNotificationConfig({
priority: "high", priority: "high",
visibility: "public", visibility: "public",
defaultSound: true, defaultSound: true,
// Only include clickAction for visible notifications (not silent ones) clickAction: `com.alertesecours.${snakeCase(actionId).toUpperCase()}`,
...(actionId && isVisible
? {
clickAction: `com.alertesecours.${snakeCase(
actionId
).toUpperCase()}`,
}
: {}),
...(android.notification || {}), ...(android.notification || {}),
}, },
restrictedPackageName: "com.alertesecours", restrictedPackageName: "com.alertesecours",
@ -94,7 +85,7 @@ function deriveNotificationConfig({
// https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.apnsconfig.md#apnsconfig_interface // https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.apnsconfig.md#apnsconfig_interface
headers: { headers: {
"apns-priority": priority === "high" ? "10" : "5", "apns-priority": priority === "high" ? "10" : "5",
"apns-push-type": isVisible ? "alert" : "background", // Use background for silent pushes "apns-push-type": "alert",
"apns-collapse-id": uid, // https://firebase.google.com/docs/cloud-messaging/concept-options "apns-collapse-id": uid, // https://firebase.google.com/docs/cloud-messaging/concept-options
...(apns.headers || {}), ...(apns.headers || {}),
}, },
@ -102,20 +93,16 @@ function deriveNotificationConfig({
aps: { aps: {
category: channel, category: channel,
threadId: channel, // Thread ID for grouping notifications threadId: channel, // Thread ID for grouping notifications
// Content available flag for background processing // Critical alerts for high importance
contentAvailable: true,
// Support for modification of notification content
mutableContent: true,
// Only include sound for non-silent notifications
...(isVisible
? {
sound: { sound: {
critical: priority === "high", critical: priority === "high",
name: "default", name: "default",
volume: 1.0, volume: 1.0,
}, },
} // Content available flag for background processing
: {}), contentAvailable: true,
// Support for modification of notification content
mutableContent: true,
// alert: { // alert: {
// // https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.apsalert.md#apsalert_interface // // https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.apsalert.md#apsalert_interface
@ -173,21 +160,14 @@ async function pushNotification({
// https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.basemessage // https://firebase.google.com/docs/reference/admin/node/firebase-admin.messaging.basemessage
const message = { const message = {
token: fcmToken, token: fcmToken,
data: { json: JSON.stringify(data), uid, actionId: notification?.actionId }, // Basic notification for platforms that don't need specific configs
notification: derivedNotification.notification,
data: { json: JSON.stringify(data), uid, actionId: notification.actionId },
// Platform specific configurations // Platform specific configurations
android: derivedNotification.android, android: derivedNotification.android,
apns: derivedNotification.apns, apns: derivedNotification.apns,
} }
// Only include notification for non-silent pushes
if (
notification &&
!notification.silent &&
(notification.title || notification.body)
) {
message.notification = derivedNotification.notification
}
try { try {
const res = await admin.messaging().send(message) const res = await admin.messaging().send(message)
logger.info( logger.info(

View file

@ -8,9 +8,7 @@ module.exports = {
SCAN_AUTO_CLOSE_CRON: "15 * * * *", // At minute 15 SCAN_AUTO_CLOSE_CRON: "15 * * * *", // At minute 15
SCAN_AUTO_ARCHIVE_CRON: "0 4 * * *", // At 4:00 SCAN_AUTO_ARCHIVE_CRON: "0 4 * * *", // At 4:00
RELATIVE_UNREGISTERED_RECONCILIATION_CRON: "0 4 * * *", // At 4:00 RELATIVE_UNREGISTERED_RECONCILIATION_CRON: "0 4 * * *", // At 4:00
DEVICE_GEODATA_IOS_SILENT_PUSH_AGE: "24 hours", // When to send iOS silent push for heartbeat sync DEVICE_GEODATA_MAX_AGE: "36 hours", // Maximum age before removal from hot storage
DEVICE_GEODATA_NOTIFICATION_AGE: "36 hours", // When to send push notification
DEVICE_GEODATA_CLEANUP_AGE: "48 hours", // When to remove/clean data
} }
// cheat on https://crontab.guru/ // cheat on https://crontab.guru/

View file

@ -10,5 +10,4 @@ module.exports = {
RELATIVE_INVITATION_NOTIFY: "relativeInvitationNotify", RELATIVE_INVITATION_NOTIFY: "relativeInvitationNotify",
ALERT_CALL_EMERGENCY_INFO_NOTIFY: "alertCallEmergencyInfoNotify", ALERT_CALL_EMERGENCY_INFO_NOTIFY: "alertCallEmergencyInfoNotify",
BACKGROUND_GEOLOCATION_LOST_NOTIFY: "backgroundGeolocationLostNotify", BACKGROUND_GEOLOCATION_LOST_NOTIFY: "backgroundGeolocationLostNotify",
IOS_GEOLOCATION_HEARTBEAT_SYNC: "iosGeolocationHeartbeatSync",
} }

View file

@ -2,24 +2,15 @@ const async = require("async")
const { ctx } = require("@modjo/core") const { ctx } = require("@modjo/core")
const ms = require("ms") const ms = require("ms")
const cron = require("~/libs/cron") const cron = require("~/libs/cron")
const { const { DEVICE_GEODATA_MAX_AGE } = require("~/constants/time")
DEVICE_GEODATA_IOS_SILENT_PUSH_AGE,
DEVICE_GEODATA_NOTIFICATION_AGE,
DEVICE_GEODATA_CLEANUP_AGE,
} = require("~/constants/time")
const tasks = require("~/tasks") const tasks = require("~/tasks")
const CLEANUP_CRON = "0 9-19 * * *" // Run every hour from 9h to 19h const CLEANUP_CRON = "0 9-19 * * *" // Run every hour from 9h to 19h
const MAX_PARALLEL_PROCESS = 10 const MAX_PARALLEL_PROCESS = 10
const COLDGEODATA_DEVICE_KEY_PREFIX = "device:geodata:" const COLDGEODATA_DEVICE_KEY_PREFIX = "device:geodata:"
const COLDGEODATA_OLD_KEY_PREFIX = "old:device:geodata:" const COLDGEODATA_OLD_KEY_PREFIX = "old:device:geodata:"
const COLDGEODATA_NOTIFIED_KEY_PREFIX = "notified:device:geodata:"
const HOTGEODATA_KEY = "device" // The key where hot geodata is stored const HOTGEODATA_KEY = "device" // The key where hot geodata is stored
const iosHeartbeatAge = Math.floor( const maxAge = Math.floor(ms(DEVICE_GEODATA_MAX_AGE) / 1000) // Convert to seconds
ms(DEVICE_GEODATA_IOS_SILENT_PUSH_AGE) / 1000
) // Convert to seconds
const notificationAge = Math.floor(ms(DEVICE_GEODATA_NOTIFICATION_AGE) / 1000) // Convert to seconds
const cleanupAge = Math.floor(ms(DEVICE_GEODATA_CLEANUP_AGE) / 1000) // Convert to seconds
module.exports = async function () { module.exports = async function () {
const logger = ctx.require("logger") const logger = ctx.require("logger")
@ -30,8 +21,8 @@ module.exports = async function () {
return async function geodataCleanupCron() { return async function geodataCleanupCron() {
logger.info("watcher geodataCleanupCron: daemon started") logger.info("watcher geodataCleanupCron: daemon started")
// Process geodata cleanup with single loop for both notifications and cleanup // Process keys in batches to avoid memory accumulation
async function processGeodataCleanup() { async function cleanupOldGeodata() {
const now = Math.floor(Date.now() / 1000) // Current time in seconds const now = Math.floor(Date.now() / 1000) // Current time in seconds
let coldCursor = "0" let coldCursor = "0"
@ -60,62 +51,25 @@ module.exports = async function () {
// Parse stored JSON to get `updatedAt` // Parse stored JSON to get `updatedAt`
const data = JSON.parse(deviceData) const data = JSON.parse(deviceData)
const age = data.updatedAt ? now - data.updatedAt : Infinity
// Handle cleanup first (48h+) - this takes priority // const age = now - data.updatedAt
if (age > cleanupAge) { const age = data.updatedAt ? now - data.updatedAt : Infinity // trick is for missing updatedAt field on old entries
// If data is older than maxAge
if (age > maxAge) {
try { try {
// Remove from hot storage // Remove from hot storage
await redisHot.zrem(HOTGEODATA_KEY, deviceId) await redisHot.zrem(HOTGEODATA_KEY, deviceId)
// Move to cleaned prefix in cold storage and clean notification flag atomically // Move to cleaned prefix in cold storage
const oldKey = `${COLDGEODATA_OLD_KEY_PREFIX}${deviceId}` const oldKey = `${COLDGEODATA_OLD_KEY_PREFIX}${deviceId}`
const notifiedKey = `${COLDGEODATA_NOTIFIED_KEY_PREFIX}${deviceId}` await redisCold.rename(key, oldKey)
const transaction = redisCold.multi()
transaction.rename(key, oldKey)
transaction.del(notifiedKey)
await transaction.exec()
logger.debug( logger.debug(
{ deviceId, age: `${Math.floor(age / 3600)}h` }, { deviceId, age: `${Math.floor(age / 3600)}h` },
"Removed old device data from hot storage and marked as cleaned in cold storage" "Removed old device data from hot storage and marked as cleaned in cold storage"
) )
} catch (error) {
logger.error(
{ error, deviceId },
"Error cleaning device data"
)
}
}
// Handle iOS silent push for heartbeat sync (24h+ but less than 36h)
else if (age > iosHeartbeatAge) {
try {
// Enqueue task to send iOS silent push for geolocation heartbeat sync
await addTask(tasks.IOS_GEOLOCATION_HEARTBEAT_SYNC, {
deviceId,
})
logger.info(
{ deviceId, age: `${Math.floor(age / 3600)}h` },
"Enqueued iOS geolocation heartbeat sync task"
)
} catch (heartbeatError) {
logger.error(
{ deviceId, error: heartbeatError },
"Error enqueueing iOS geolocation heartbeat sync task"
)
}
}
// Handle notification (36h+ but less than 48h)
else if (age > notificationAge) {
const notifiedKey = `${COLDGEODATA_NOTIFIED_KEY_PREFIX}${deviceId}`
try {
// Check if we've already notified for this device
const alreadyNotified = await redisCold.exists(notifiedKey)
if (!alreadyNotified) {
// Enqueue task to notify user about lost background geolocation // Enqueue task to notify user about lost background geolocation
try { try {
await addTask(tasks.BACKGROUND_GEOLOCATION_LOST_NOTIFY, { await addTask(tasks.BACKGROUND_GEOLOCATION_LOST_NOTIFY, {
@ -123,7 +77,7 @@ module.exports = async function () {
}) })
logger.info( logger.info(
{ deviceId, age: `${Math.floor(age / 3600)}h` }, { deviceId },
"Enqueued background geolocation lost notification task" "Enqueued background geolocation lost notification task"
) )
} catch (notifError) { } catch (notifError) {
@ -132,18 +86,18 @@ module.exports = async function () {
"Error enqueueing background geolocation lost notification task" "Error enqueueing background geolocation lost notification task"
) )
} }
// Mark as notified with 48h expiry (cleanup age)
await redisCold.set(notifiedKey, "1", "EX", cleanupAge)
}
} catch (error) { } catch (error) {
logger.error( logger.error(
{ error, deviceId }, { error, deviceId },
"Error processing notification for device" "Error cleaning device data"
) )
} }
} }
} catch (error) { } catch (error) {
logger.error({ error, key }, "Error processing device data") logger.error(
{ error, key },
"Error processing device data from cold storage"
)
} }
}) })
} }
@ -151,58 +105,58 @@ module.exports = async function () {
} }
// this is temporary function (fixing actual data) // this is temporary function (fixing actual data)
// async function cleanupOrphanedHotGeodata() { async function cleanupOrphanedHotGeodata() {
// // Get all devices from hot storage // Get all devices from hot storage
// const hotDevices = new Set() const hotDevices = new Set()
// let hotCursor = "0" let hotCursor = "0"
// do { do {
// // Use zscan to iterate through the sorted set // Use zscan to iterate through the sorted set
// const [newCursor, items] = await redisHot.zscan( const [newCursor, items] = await redisHot.zscan(
// HOTGEODATA_KEY, HOTGEODATA_KEY,
// hotCursor, hotCursor,
// "COUNT", "COUNT",
// "100" "100"
// ) )
// hotCursor = newCursor hotCursor = newCursor
// // Extract device IDs (every other item in the result is a score) // Extract device IDs (every other item in the result is a score)
// for (let i = 0; i < items.length; i += 2) { for (let i = 0; i < items.length; i += 2) {
// hotDevices.add(items[i]) hotDevices.add(items[i])
// } }
// } while (hotCursor !== "0") } while (hotCursor !== "0")
// // Process each hot device // Process each hot device
// await async.eachLimit( await async.eachLimit(
// [...hotDevices], [...hotDevices],
// MAX_PARALLEL_PROCESS, MAX_PARALLEL_PROCESS,
// async (deviceId) => { async (deviceId) => {
// try { try {
// // Check if device exists in cold storage // Check if device exists in cold storage
// const coldKey = `${COLDGEODATA_DEVICE_KEY_PREFIX}${deviceId}` const coldKey = `${COLDGEODATA_DEVICE_KEY_PREFIX}${deviceId}`
// const exists = await redisCold.exists(coldKey) const exists = await redisCold.exists(coldKey)
// // If device doesn't exist in cold storage, remove it from hot storage // If device doesn't exist in cold storage, remove it from hot storage
// if (!exists) { if (!exists) {
// await redisHot.zrem(HOTGEODATA_KEY, deviceId) await redisHot.zrem(HOTGEODATA_KEY, deviceId)
// logger.debug( logger.debug(
// { deviceId }, { deviceId },
// "Removed orphaned device data from hot storage (not found in cold storage)" "Removed orphaned device data from hot storage (not found in cold storage)"
// ) )
// } }
// } catch (error) { } catch (error) {
// logger.error( logger.error(
// { error, deviceId }, { error, deviceId },
// "Error checking orphaned device data" "Error checking orphaned device data"
// ) )
// } }
// } }
// ) )
// } }
// Schedule cleanup function to run periodically // Schedule both cleanup functions to run periodically
cron.schedule(CLEANUP_CRON, async () => { cron.schedule(CLEANUP_CRON, async () => {
await processGeodataCleanup() await cleanupOldGeodata()
// await cleanupOrphanedHotGeodata() await cleanupOrphanedHotGeodata()
}) })
} }
} }