feat: Extend crawler logging

jest.config.js

@@ -10,8 +10,8 @@ export default {
   coverageReporters: ['json-summary', 'text'],
   coverageThreshold: {
     global: {
-      lines: 71.6,
-      statements: 71.63,
+      lines: 71.8,
+      statements: 71.83,
       branches: 63.3,
       functions: 69.72,
     },

src/crawler.ts

@@ -23,7 +23,7 @@ export class Crawler {
     const registrations =
       await this.registrationStore.findRegistrationsReadBefore(dateLastRead);
     for (const registration of registrations) {
-      this.logger.info(`Crawling registration URL ${registration.url}`);
+      this.logger.info(`Crawling registration URL ${registration.url}...`);
       let datasets: DatasetExt[] = [];
       let statusCode = 200;
       let isValid = false;
@@ -33,21 +33,28 @@ export class Crawler {
         const validationResult = await this.validator.validate(data);
         isValid = validationResult.state === 'valid';
         if (isValid) {
+          this.logger.info(`${registration.url} passes validation`);
           datasets = await fetch(registration.url);
           await this.datasetStore.store(datasets);
           datasets.map(async dataset => {
             const dcatValidationResult = await this.validator.validate(dataset);
             const rating = rate(dcatValidationResult as Valid);
             await this.ratingStore.store(extractIri(dataset), rating);
           });
+        } else {
+          this.logger.info(`${registration.url} does not pass validation`);
         }
       } catch (e) {
         if (e instanceof HttpError) {
           statusCode = e.statusCode;
+          this.logger.info(
+            `${registration.url} returned HTTP error ${statusCode}`
+          );
         }
 
         if (e instanceof NoDatasetFoundAtUrl) {
           // Request was successful, but no datasets exist any longer at the URL.
+          this.logger.info(`${registration.url} has no datasets`);
         }
       }
 

src/graphdb.ts

@@ -154,7 +154,7 @@ export class GraphDbRegistrationStore implements RegistrationStore {
 
   constructor(private client: GraphDbClient) {}
 
-  async store(registration: Registration) {
+  async store(registration: Registration): Promise<void> {
     const quads = [
       this.registrationQuad(
         registration,
@@ -268,7 +268,7 @@ export class GraphDbRegistrationStore implements RegistrationStore {
             url: '/statements',
             body: result,
           });
-          resolve(null);
+          resolve();
         } catch (e) {
           reject(e);
         }

src/registration.ts

@@ -52,7 +52,7 @@ export interface RegistrationStore {
   /**
    * Store a {@see Registration}, replacing any Registrations with the same URL.
    */
-  store(registration: Registration): void;
+  store(registration: Registration): Promise<void>;
   findRegistrationsReadBefore(date: Date): Promise<Registration[]>;
 }
 

test/mock.ts

@@ -31,7 +31,7 @@ export class MockRegistrationStore implements RegistrationStore {
     );
   }
 
-  store(registration: Registration): void {
+  async store(registration: Registration): Promise<void> {
     this.registrations.set(registration.url, registration);
   }
 }