diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4fff1098ca10d2e60d1c4fdc41f4f964f38721d..ae00c9a6c86ffedc16c1d59d8abbcbba6b0b476d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,7 +48,7 @@ lint: stage: Lint image: node:latest script: - - npm install -g eslint + - npm install -g eslint@7 - eslint --init - npm run lint rules: diff --git a/src/app.ts b/src/app.ts index 2c862caae982fdcfb15776da349c4235d8256336..ebf340e6824a407ef1a9af69952919ee49a103a7 100644 --- a/src/app.ts +++ b/src/app.ts @@ -40,17 +40,17 @@ const statOrigins = ['http://localhost', 'https://localhost:3000']; const origins = 'CORS_ORIGINS_ALLOWED' in process.env ? process.env.CORS_ORIGINS_ALLOWED.split(',') - .map(origin => { - try { - const originURL = new URL(origin); - return originURL.origin.endsWith('.cern.ch') ? origin : undefined; - } catch { - console.debug('Invalid origin: ', origin); - return undefined; - } - }) - .filter(origin => origin !== undefined) - .concat(statOrigins) + .map(origin => { + try { + const originURL = new URL(origin); + return originURL.origin.endsWith('.cern.ch') ? origin : undefined; + } catch { + console.debug('Invalid origin: ', origin); + return undefined; + } + }) + .filter(origin => origin !== undefined) + .concat(statOrigins) : statOrigins; //create express server @@ -63,6 +63,10 @@ sentry.sentryHandleRequests(app); // eslint-disable-next-line @typescript-eslint/no-var-requires app.use(require('express-status-monitor')()); +// Nicely formatted json (debug only) +if (process.env.NODE_ENV == 'development') + app.set('json spaces', 2) + let server; // Parse class-validator classes into JSON Schema: diff --git a/src/models/cern-activedirectory.ts b/src/models/cern-activedirectory.ts index 69a6172c5f7870c2f88b1328214b70c8f66bd315..4edea6f388701727d4dcb19d4ddf22552a04dfad 100644 --- a/src/models/cern-activedirectory.ts +++ b/src/models/cern-activedirectory.ts @@ -147,4 +147,33 @@ export class CERNActiveDirectory { ADGroup.owner && ADGroup.owner.replace('CN=', '').split(',')[0], ); } + + static async getUserDepartment(mailTarget: string) { + const filter = ldapEscape.filter`(&\ +(|(mail=${mailTarget})(proxyAddresses=smtp:${mailTarget}))\ +(objectClass=user)\ +(|\ +(memberOf=CN=cern-accounts-primary,OU=e-groups,OU=Workgroups,DC=cern,DC=ch)\ +(memberOf=CN=cern-accounts-secondary,OU=e-groups,OU=Workgroups,DC=cern,DC=ch)\ +(memberOf=CN=cern-accounts-service,OU=e-groups,OU=Workgroups,DC=cern,DC=ch)\ +))`; + const opts = { + filter: filter, + attributes: ['cn', 'mail', 'division'], + sizeLimit: 1, + }; + + const ad = new ActiveDirectory.promiseWrapper(CERN_AD_config); + + const result = await ad.find(opts).catch(err => { + console.error('CERNActiveDirectory.getUserDepartment: ERROR: ' + err); + throw err; + }); + + if (result) { + if (result.users.length > 0) { + return result.users[0].division; + } + } + } } diff --git a/src/models/channel.ts b/src/models/channel.ts index c695d8a6b75d5fdc3200aab591d5cf5d01304abf..1e8d48af613eeeedcebe78e1ae1769ca8bd1e135 100644 --- a/src/models/channel.ts +++ b/src/models/channel.ts @@ -17,6 +17,7 @@ import { OneToMany, PrimaryGeneratedColumn, Raw, + RelationId, } from 'typeorm'; import { IsEmail, IsOptional, IsUrl, Validate } from 'class-validator'; import { User } from './user'; @@ -65,6 +66,9 @@ export class Channel extends ApiKeyObject { @ManyToOne(type => User, user => user.ownedChannels) owner: User; + @RelationId((channel: Channel) => channel.owner) + ownerId: string; + @Validate(AlphaNumericPunctuationChannelName) @Index({ unique: true }) @Column() @@ -101,6 +105,7 @@ export class Channel extends ApiKeyObject { @OneToMany(type => Notification, notification => notification.target, { cascade: true, + onDelete: 'CASCADE', }) @Type(() => Notification) notifications: Notification[]; diff --git a/src/models/device.ts b/src/models/device.ts index d0421eb72482a67f008956a3d4ee0e2acdceec92..139891a13de05cccf9134a0e940dcee3031870ee 100644 --- a/src/models/device.ts +++ b/src/models/device.ts @@ -1,4 +1,4 @@ -import { Entity, ManyToOne, PrimaryGeneratedColumn, Column, BeforeInsert, BeforeUpdate } from 'typeorm'; +import { Entity, ManyToOne, PrimaryGeneratedColumn, Column, BeforeInsert, BeforeUpdate, RelationId } from 'typeorm'; import { User } from './user'; export enum DeviceType { @@ -43,6 +43,9 @@ export class Device { @ManyToOne(type => User, user => user.devices) user: User; + @RelationId((device: Device) => device.user) + userId: string; + @Column({ nullable: true }) info: string; diff --git a/src/models/mute.ts b/src/models/mute.ts index 6ad0ab9699f084449152ac69a778bcc1ee8be54d..f49d06e1e9090ce6b487bf8adf30c98dc92130d3 100644 --- a/src/models/mute.ts +++ b/src/models/mute.ts @@ -1,9 +1,10 @@ -import { Entity, ManyToOne, PrimaryGeneratedColumn, Column } from 'typeorm'; +import { Entity, ManyToOne, PrimaryGeneratedColumn, Column, RelationId } from 'typeorm'; import { User } from './user'; import { Channel } from './channel'; import { BadRequestError } from 'routing-controllers'; import { EntityManager } from 'typeorm'; import { AESCypher } from '../middleware/aes-cipher'; +import { Device } from './device'; @Entity('Mutes') export class Mute { @@ -13,6 +14,9 @@ export class Mute { @ManyToOne(type => User, user => user.mutes) user: User; + @RelationId((device: Device) => device.user) + userId: string; + @ManyToOne(type => Channel, { nullable: true }) target: Channel; diff --git a/src/models/notification.ts b/src/models/notification.ts index 1950e49ec8d832be640d74f5b50b07d8c430c58a..edb97c8f207b771e75276b269eb8b5b83f746122 100644 --- a/src/models/notification.ts +++ b/src/models/notification.ts @@ -8,6 +8,7 @@ import { Index, Brackets, DeleteDateColumn, + RelationId, } from 'typeorm'; import { Channel } from './channel'; import { User } from './user'; @@ -16,6 +17,7 @@ import { AuthorizationBag } from './authorization-bag'; import { Source, PriorityLevel } from './notification-enums'; import { Group as AuthGroup } from './cern-authorization-service'; import { Type } from 'class-transformer'; +import { Device } from './device'; @Entity({ name: 'Notifications' }) export class Notification { @@ -36,6 +38,9 @@ export class Notification { @ManyToOne(type => Channel, channel => channel.notifications) target: Channel; + @RelationId((notification: Notification) => notification.target) + targetId: string; + @Column({ nullable: true }) link: string; @Column({ nullable: true }) summary: string; @Column({ nullable: true }) imgUrl: string; diff --git a/src/models/preference.ts b/src/models/preference.ts index d33504eb73bca3a6861e06f5614747c6ec636091..ac09f0b8d7d6c92d9372258dc7ae49263e44af39 100644 --- a/src/models/preference.ts +++ b/src/models/preference.ts @@ -9,6 +9,7 @@ import { BeforeInsert, BeforeUpdate, AfterLoad, + RelationId, } from 'typeorm'; import { User } from './user'; import { Channel } from './channel'; @@ -39,6 +40,9 @@ export class Preference { @ManyToOne(type => User, user => user.preferences) user: User; + @RelationId((device: Device) => device.user) + userId: string; + @ManyToOne(type => Channel, { nullable: true }) target: Channel; @@ -62,7 +66,7 @@ export class Preference { @BeforeInsert() async priorityToUpperCase(): Promise<void> { if (!this.notificationPriority) return; - for (var i = 0; i < this.notificationPriority.length; i++) + for (let i = 0; i < this.notificationPriority.length; i++) this.notificationPriority[i] = this.notificationPriority[i].toUpperCase() as PriorityLevel; } diff --git a/src/models/statistics.ts b/src/models/statistics.ts index 3876c9b6040f01c8d19f618f7db3f023e4e8be4f..5300af1e1b32e9ead854a400b3b4561c5e9e67c1 100644 --- a/src/models/statistics.ts +++ b/src/models/statistics.ts @@ -1,13 +1,3 @@ -import { EntityManager } from 'typeorm'; -import { Channel } from './channel'; -import { Notification } from './notification'; -import { User } from './user'; -import { Device } from './device'; -import { Preference } from './preference'; -import { Mute } from './mute'; -import { Category } from './category'; -import { Tag } from './tag'; - type LastDayStatistics = { channels_created: number; notifications_sent: number; @@ -27,69 +17,24 @@ export class Statistics { mutes: number; categories: number; tags: number; - lastday: LastDayStatistics = <LastDayStatistics>{}; + last_day: LastDayStatistics = <LastDayStatistics>{}; + + top_channels_with_self_subscription: Record<string, number>; + top_channels_with_notifications: Record<string, number>; + top_latest_created_channels: Record<string, Date>; + avg_notifications_per_channel: number; - async buildStatistics(transactionManager: EntityManager) { - // Global numbers (total) - //this.channels = await transactionManager.getRepository(Channel).count(); - this.channels = await transactionManager - .getRepository(Channel) - .createQueryBuilder('channel') - .where("visibility != 'PRIVATE'") - .getCount(); - this.channels_public = await transactionManager - .getRepository(Channel) - .createQueryBuilder('channel') - .where("visibility = 'PUBLIC'") - .getCount(); - this.channels_restricted = await transactionManager - .getRepository(Channel) - .createQueryBuilder('channel') - .where("visibility = 'RESTRICTED'") - .getCount(); - this.channels_internal = await transactionManager - .getRepository(Channel) - .createQueryBuilder('channel') - .where("visibility = 'INTERNAL'") - .getCount(); - this.notifications = await transactionManager.getRepository(Notification).count(); - this.users = await transactionManager.getRepository(User).count(); - this.users_active = await transactionManager - .getRepository(User) - .createQueryBuilder('user') - .where('"lastLogin" IS NOT NULL') - .getCount(); - this.devices = await transactionManager - .getRepository(Device) - .createQueryBuilder('device') - .where("type != 'MAIL'") - .getCount(); - this.preferences = await transactionManager - .getRepository(Preference) - .createQueryBuilder('preference') - .where("name != 'Default Live' and name != 'Default Daily'") - .getCount(); - this.mutes = await transactionManager.getRepository(Mute).count(); - this.categories = await transactionManager.getRepository(Category).count(); - this.tags = await transactionManager.getRepository(Tag).count(); + top_max_devices: Record<string, number>; + top_max_preferences: Record<string, number>; + top_max_mutes: Record<string, number>; + avg_devices_per_user: number; + avg_preferences_per_user: number; + avg_mutes_per_user: number; - // Daily numbers (last 24h) - this.lastday.channels_created = await transactionManager - .getRepository(Channel) - .createQueryBuilder('channel') - .where('"creationDate" > NOW() - INTERVAL \'1 DAY\'') - .getCount(); - this.lastday.notifications_sent = await transactionManager - .getRepository(Notification) - .createQueryBuilder('notification') - .where('"sentAt" > NOW() - INTERVAL \'1 DAY\'') - .getCount(); - this.lastday.users_active = await transactionManager - .getRepository(User) - .createQueryBuilder('user') - .where('"lastLogin" > NOW() - INTERVAL \'1 DAY\'') - .getCount(); + active_users_per_department: Record<string, number>; + channels_per_department: Record<string, number>; - console.debug(this); - } + last_notifications_reach: Record<string, Record<string, number>>; + avg_users_reached_per_notification_weekly: number; + avg_devices_reached_per_notification: number; } diff --git a/src/services/impl/statistics/get-statistics.ts b/src/services/impl/statistics/get-statistics.ts index 44c2cf3e2d99f488d270b83e9fee8822488cd2bf..344eb76298cb11368768b53bd3727fbc9384637e 100644 --- a/src/services/impl/statistics/get-statistics.ts +++ b/src/services/impl/statistics/get-statistics.ts @@ -1,18 +1,349 @@ -import { Command } from "../command"; -import { EntityManager } from "typeorm"; -import { Statistics } from "../../../models/statistics"; +import { Command } from '../command'; +import { EntityManager } from 'typeorm'; +import { Statistics } from '../../../models/statistics'; +import { Channel } from '../../../models/channel'; +import { Notification } from '../../../models/notification'; +import { User } from '../../../models/user'; +import { Device } from '../../../models/device'; +import { Preference } from '../../../models/preference'; +import { Mute } from '../../../models/mute'; +import { Category } from '../../../models/category'; +import { Tag } from '../../../models/tag'; +import { CERNActiveDirectory } from '../../../models/cern-activedirectory'; +import { AuditNotifications } from '../../../log/auditing'; export class GetStatistics implements Command { - constructor() { } + s = new Statistics(); async execute(transactionManager: EntityManager): Promise<Statistics> { try { - let s = new Statistics(); - await s.buildStatistics(transactionManager); - return s; + await this.buildStatistics(transactionManager); + return this.s; } catch (ex) { - console.error("Error GetStatistics", ex.message); - return null as Statistics; + console.error('Error GetStatistics', ex.message); + return null as Statistics; + } + } + + async buildStatistics(transactionManager: EntityManager) { + // Global numbers (total) + //this.channels = await transactionManager.getRepository(Channel).count(); + this.s.channels = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .where("visibility != 'PRIVATE'") + .getCount(); + this.s.channels_public = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .where("visibility = 'PUBLIC'") + .getCount(); + this.s.channels_restricted = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .where("visibility = 'RESTRICTED'") + .getCount(); + this.s.channels_internal = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .where("visibility = 'INTERNAL'") + .getCount(); + this.s.notifications = await transactionManager.getRepository(Notification).count(); + this.s.users = await transactionManager.getRepository(User).count(); + this.s.devices = await transactionManager + .getRepository(Device) + .createQueryBuilder('device') + .where("type != 'MAIL'") + .getCount(); + this.s.preferences = await transactionManager + .getRepository(Preference) + .createQueryBuilder('preference') + .where("name != 'Default Live' and name != 'Default Daily'") + .getCount(); + this.s.mutes = await transactionManager.getRepository(Mute).count(); + this.s.categories = await transactionManager.getRepository(Category).count(); + this.s.tags = await transactionManager.getRepository(Tag).count(); + + // Daily numbers (last 24h) + this.s.last_day.channels_created = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .where('"creationDate" > NOW() - INTERVAL \'1 DAY\'') + .getCount(); + this.s.last_day.notifications_sent = await transactionManager + .getRepository(Notification) + .createQueryBuilder('notification') + .where('"sentAt" > NOW() - INTERVAL \'1 DAY\'') + .getCount(); + this.s.last_day.users_active = await transactionManager + .getRepository(User) + .createQueryBuilder('user') + .where('"lastLogin" > NOW() - INTERVAL \'1 DAY\'') + .getCount(); + + await this.calculateDeviceStats(transactionManager); + await this.calculateMuteStats(transactionManager); + await this.calculatePreferenceStats(transactionManager); + await this.calculateUserStats(transactionManager); + await this.calculateNotificationStats(transactionManager); + await this.calculateChannelStats(transactionManager); + } + + async calculateChannelStats(transactionManager: EntityManager) { + const owners = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .leftJoinAndSelect('channel.owner', 'owner') + .distinctOn(['owner.email']) + .select('owner.email') + .getRawMany(); + + // channels per department + this.s.channels_per_department = {}; + for (const owner of owners) { + let ownerDepartment = await CERNActiveDirectory.getUserDepartment(owner.owner_email); + if (!ownerDepartment) ownerDepartment = '-'; + if (ownerDepartment in this.s.channels_per_department) + this.s.channels_per_department[ownerDepartment] = this.s.channels_per_department[ownerDepartment] + 1; + else this.s.channels_per_department[ownerDepartment] = 1; + } + + const channels = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .select(['channel.name', 'channel.creationDate']) + .orderBy('channel.creationDate', 'DESC') + .limit(5) + .getRawMany(); + + this.s.top_latest_created_channels = {}; + channels.map(item => { + this.s.top_latest_created_channels[item.channel_name] = item.channel_name; + }); + } + + async calculateDeviceStats(transactionManager: EntityManager) { + const maxDevices = await transactionManager + .getRepository(Device) + .createQueryBuilder('device') + .select('COUNT(device.id) AS device_count') + .groupBy('device.userId') + .orderBy('device_count', 'DESC') + .limit(5) + .getRawMany(); + + this.s.top_max_devices = {}; + maxDevices.map((item, index) => { + this.s.top_max_devices['***' + index] = item.device_count; + }); + + const device_avg = await transactionManager + .createQueryBuilder() + .select('AVG(device_count) AS average') + .from( + qb => + qb + .select('COUNT(device.id) AS device_count') + .from(Device, 'device') + .groupBy('device.userId') + .orderBy('device_count', 'DESC') + .select('COUNT(device.id) AS device_count'), + + 'device_counts', + ) + .getRawOne(); + + this.s.avg_devices_per_user = device_avg.average; + } + + async calculatePreferenceStats(transactionManager: EntityManager) { + const maxPreferences = await transactionManager + .getRepository(Preference) + .createQueryBuilder('preference') + .select('COUNT(preference.id) AS preference_count') + .groupBy('preference.userId') + .orderBy('preference_count', 'DESC') + .limit(5) + .getRawMany(); + + this.s.top_max_preferences = {}; + maxPreferences.map((item, index) => { + this.s.top_max_preferences['***' + index] = item.preference_count; + }); + + const preference_avg = await transactionManager + .createQueryBuilder() + .select('AVG(preference_count) AS average') + .from( + qb => + qb + .select('COUNT(preference.id) AS preference_count') + .from(Device, 'preference') + .groupBy('preference.userId') + .orderBy('preference_count', 'DESC') + .select('COUNT(preference.id) AS preference_count'), + + 'preference_counts', + ) + .getRawOne(); + + this.s.avg_preferences_per_user = preference_avg.average; + } + + async calculateMuteStats(transactionManager: EntityManager) { + const max = await transactionManager + .getRepository(Mute) + .createQueryBuilder('mute') + .select('COUNT(mute.id) AS mute_count') + .groupBy('mute.userId') + .orderBy('mute_count', 'DESC') + .limit(5) + .getRawMany(); + + this.s.top_max_mutes = {}; + max.map((item, index) => { + this.s.top_max_mutes['***' + index] = item.mute_count; + }); + + const avgs = await transactionManager + .createQueryBuilder() + .select('AVG(mute_count) AS average') + .from( + qb => + qb + .select('COUNT(mute.id) AS mute_count') + .from(Device, 'mute') + .groupBy('mute.userId') + .orderBy('mute_count', 'DESC') + .select('COUNT(mute.id) AS mute_count'), + + 'mute_counts', + ) + .getRawOne(); + + this.s.avg_mutes_per_user = avgs.average; + } + + async calculateUserStats(transactionManager: EntityManager) { + const maxMembers = await transactionManager + .getRepository(Channel) + .createQueryBuilder('channel') + .leftJoinAndSelect('channel.members', 'member') + .select(['COUNT(channel.id) AS users_count', 'channel.id', 'channel.name']) + .groupBy('channel.id') + .addGroupBy('channel.name') + .orderBy('users_count', 'DESC') + .limit(5) + .getRawMany(); + + this.s.top_channels_with_self_subscription = {}; + maxMembers.map((item, index) => { + this.s.top_channels_with_self_subscription[item.channel_name] = item.users_count; + }); + + const arrayActiveUser = await transactionManager + .getRepository(User) + .createQueryBuilder('user') + .select(['user.email', 'user.lastLogin']) + .where('user.lastLogin IS NOT NULL') + .getRawMany(); + + this.s.users_active = arrayActiveUser.length; + this.s.active_users_per_department = {}; + for (const user of arrayActiveUser) { + let userDepartment = await CERNActiveDirectory.getUserDepartment(user.user_email); + if (!userDepartment) userDepartment = '-'; + if (userDepartment in this.s.active_users_per_department) + this.s.active_users_per_department[userDepartment] = this.s.active_users_per_department[userDepartment] + 1; + else this.s.active_users_per_department[userDepartment] = 1; + } + } + + async calculateNotificationStats(transactionManager: EntityManager) { + const maxNotifications = await transactionManager + .getRepository(Notification) + .createQueryBuilder('notification') + .select(['COUNT(notification.id) AS notification_count', 'channel.name']) + .innerJoinAndSelect('notification.target', 'channel') + .where('channel.deleteDate is NULL') + .groupBy('notification.targetId') + .addGroupBy('channel.name') + .addGroupBy('channel.id') + .orderBy('notification_count', 'DESC') + .limit(5) + .getRawMany(); + + this.s.top_channels_with_notifications = {}; + maxNotifications.map((item, index) => { + this.s.top_channels_with_notifications[item.channel_name] = item.notification_count; + }); + + const notification_avg = await transactionManager + .createQueryBuilder() + .select('AVG(notification_count) AS average') + .from( + qb => + qb + .select('COUNT(notification.id) AS notification_count') + .from(Notification, 'notification') + .groupBy('notification.targetId') + .orderBy('notification_count', 'DESC'), + 'notification_counts', + ) + .getRawOne(); + + this.s.avg_notifications_per_channel = notification_avg.average; + + const arrayNotification = await transactionManager + .getRepository(Notification) + .createQueryBuilder('notification') + .where("notification.sentAt >= now() - interval '1 week'") + .select(['notification.id']) + .orderBy('notification.sentAt', 'DESC') + .getRawMany(); + + this.s.last_notifications_reach = {}; + let count = 0; + this.s.avg_users_reached_per_notification_weekly = 0; + this.s.avg_devices_reached_per_notification = 0; + for (const notification of arrayNotification) { + const anon_id = '***-' + count; // anonymize notification id + this.s.last_notifications_reach[anon_id] = {}; + try { + const oneAudit = (await AuditNotifications.getValuesAsJson(notification.notification_id)) as any; + if ( + oneAudit && + notification.notification_id in oneAudit && + oneAudit[notification.notification_id].router && + oneAudit[notification.notification_id].router.target_users + ) { + if (count < 10) + this.s.last_notifications_reach[anon_id]['target_users'] = Object.keys( + oneAudit[notification.notification_id].router.target_users, + ).length; + this.s.avg_users_reached_per_notification_weekly += Object.keys( + oneAudit[notification.notification_id].router.target_users, + ).length; + this.s.last_notifications_reach[anon_id]['devices'] = 0; + for (const oneTargetUser of Object.keys(oneAudit[notification.notification_id].router.target_users)) { + if (count < 10) + this.s.last_notifications_reach[anon_id]['devices'] += Object.keys( + oneAudit[notification.notification_id].router.target_users[oneTargetUser], + ).length; + this.s.avg_devices_reached_per_notification += Object.keys( + oneAudit[notification.notification_id].router.target_users[oneTargetUser], + ).length; + } + count++; + } + // eslint-disable-next-line no-empty + } catch (NotFoundError) {} + } + if (arrayNotification.length > 0) { + this.s.avg_users_reached_per_notification_weekly = + this.s.avg_users_reached_per_notification_weekly / arrayNotification.length; + this.s.avg_devices_reached_per_notification = + this.s.avg_devices_reached_per_notification / arrayNotification.length; } } }