fix(maintenance): prevent enable/disable maintenance CLI hanging on occasion (#24713)

* fix(maintenance): prevent CLI hanging on occassion
fix(maintenance): always ack messages
fix(maintenance): ensure Redis is connected first

* chore(maintenance): validate app restart responses

* chore: mock the app restart callback

* fix: ack may not exist depending on caller

* refactor: move one shot into app.repository

* fix: send correct state in one shot

* chore: log restart event

server/src/repositories/app.repository.ts

@@ -1,5 +1,10 @@
 import { Injectable } from '@nestjs/common';
+import { createAdapter } from '@socket.io/redis-adapter';
+import Redis from 'ioredis';
+import { Server as SocketIO } from 'socket.io';
 import { ExitCode } from 'src/enum';
+import { ConfigRepository } from 'src/repositories/config.repository';
+import { AppRestartEvent } from 'src/repositories/event.repository';
 
 @Injectable()
 export class AppRepository {
@@ -17,4 +22,26 @@ export class AppRepository {
   setCloseFn(fn: () => Promise<void>) {
     this.closeFn = fn;
   }
+
+  async sendOneShotAppRestart(state: AppRestartEvent): Promise<void> {
+    const server = new SocketIO();
+    const { redis } = new ConfigRepository().getEnv();
+    const pubClient = new Redis({ ...redis, lazyConnect: true });
+    const subClient = pubClient.duplicate();
+
+    await Promise.all([pubClient.connect(), subClient.connect()]);
+
+    server.adapter(createAdapter(pubClient, subClient));
+
+    // => corresponds to notification.service.ts#onAppRestart
+    server.emit('AppRestartV1', state, async () => {
+      const responses = await server.serverSideEmitWithAck('AppRestart', state);
+      if (responses.some((response) => response !== 'ok')) {
+        throw new Error("One or more node(s) returned a non-'ok' response to our restart request!");
+      }
+
+      pubClient.disconnect();
+      subClient.disconnect();
+    });
+  }
 }