pocketbase + script changes

scripts/ingest/upload-visits-data.js

@@ -0,0 +1,137 @@
+import PocketBase from 'pocketbase';
+import fs from 'fs';
+import path from 'path';
+
+async function uploadVisitsData() {
+  const pb = new PocketBase('http://localhost:8090');
+
+  // Authenticate as superuser
+  try {
+    await pb.collection("_superusers").authWithPassword('azeem.fidahusein@gmail.com', 'azsxdcazsxdc');
+    console.log('Authenticated as superuser');
+  } catch (error) {
+    console.error('Failed to authenticate as superuser:', error.message);
+    throw error;
+  }
+
+  // Read and parse CSV file
+  const csvPath = '/Users/azeem/repos/personal-tracker/data/timeline_csv/visits.csv';
+
+  try {
+    const csvData = fs.readFileSync(csvPath, 'utf8');
+    const lines = csvData.trim().split('\n');
+    const headers = lines[0].split(',').map(h => h.trim().replace(/\r$/, ''));
+
+    console.log(`Found ${lines.length - 1} rows to upload`);
+    console.log('Headers:', headers);
+
+    let successCount = 0;
+    let errorCount = 0;
+
+    // Process each row (skip header)
+    for (let i = 1; i < lines.length; i++) {
+      const values = parseCsvLine(lines[i]);
+
+      if (values.length !== headers.length) {
+        console.warn(`Row ${i}: Column count mismatch, skipping`);
+        errorCount++;
+        continue;
+      }
+
+      // Create record object
+      const record = {};
+      headers.forEach((header, index) => {
+        const value = values[index];
+
+        // Handle different data types based on field name
+        switch (header) {
+          case 'segment_index':
+          case 'hierarchyLevel':
+            record[header] = value !== '' ? parseInt(value) : null;
+            break;
+          case 'visit_probability':
+          case 'top_probability':
+          case 'top_lat':
+          case 'top_lon':
+            record[header] = value !== '' ? parseFloat(value) : null;
+            break;
+          case 'startTime':
+          case 'endTime':
+            // Convert ISO string to proper date format
+            record[header] = value ? new Date(value).toISOString() : null;
+            break;
+          default:
+            record[header] = value || null;
+        }
+      });
+
+      try {
+        await pb.collection('visits').create(record);
+        successCount++;
+
+        if (successCount % 100 === 0) {
+          console.log(`Uploaded ${successCount} records...`);
+        }
+      } catch (error) {
+        console.error(`Row ${i} failed:`, error.message);
+        if (error.response && error.response.data) {
+          console.error(`PocketBase error:`, JSON.stringify(error.response.data, null, 2));
+        }
+        console.error(`Record data:`, JSON.stringify(record, null, 2));
+        errorCount++;
+
+        // Stop after first few errors to debug
+        if (errorCount >= 3) {
+          console.log('Stopping after 3 errors to debug...');
+          break;
+        }
+      }
+    }
+
+    console.log(`\nUpload completed:`);
+    console.log(`Success: ${successCount} records`);
+    console.log(`Errors: ${errorCount} records`);
+
+  } catch (error) {
+    console.error('Error reading CSV file:', error.message);
+    throw error;
+  }
+}
+
+// Helper function to parse CSV line properly (handles commas in quoted fields)
+function parseCsvLine(line) {
+  const result = [];
+  let current = '';
+  let inQuotes = false;
+
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+
+    if (char === '"') {
+      inQuotes = !inQuotes;
+    } else if (char === ',' && !inQuotes) {
+      result.push(current.trim());
+      current = '';
+    } else {
+      current += char;
+    }
+  }
+
+  result.push(current.trim());
+  return result;
+}
+
+// Check if this file is being run directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  uploadVisitsData()
+    .then(() => {
+      console.log('Upload script completed successfully');
+      process.exit(0);
+    })
+    .catch((error) => {
+      console.error('Upload script failed:', error);
+      process.exit(1);
+    });
+}
+
+export { uploadVisitsData };