fix: Improve looping performance (#23541)

* Use herdsman improved looping.

* Fix lint.

* Fix tests.

* Feedback

Author: Nerivec

lib/controller.ts

@@ -157,15 +157,19 @@ export class Controller {
         }
 
         // Log zigbee clients on startup
-        const devices = this.zigbee.devices(false);
-        logger.info(`Currently ${devices.length} devices are joined:`);
-        for (const device of devices) {
+        let deviceCount = 0;
+
+        for (const device of this.zigbee.devicesIterator(utils.deviceNotCoordinator)) {
             const model = device.isSupported
                 ? `${device.definition.model} - ${device.definition.vendor} ${device.definition.description}`
                 : 'Not supported';
             logger.info(`${device.name} (${device.ieeeAddr}): ${model} (${device.zh.type})`);
+
+            deviceCount++;
         }
 
+        logger.info(`Currently ${deviceCount} devices are joined.`);
+
         // Enable zigbee join
         try {
             if (settings.get().permit_join) {
@@ -193,7 +197,7 @@ export class Controller {
 
         // Send all cached states.
         if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
-            for (const entity of [...devices, ...this.zigbee.groups()]) {
+            for (const entity of this.zigbee.devicesAndGroupsIterator()) {
                 if (this.state.exists(entity)) {
                     await this.publishEntityState(entity, this.state.get(entity), 'publishCached');
                 }

lib/extension/availability.ts

@@ -140,7 +140,7 @@ export default class Availability extends Extension {
         await this.publishAvailabilityForAllEntities();
 
         // Start availability for the devices
-        for (const device of this.zigbee.devices(false)) {
+        for (const device of this.zigbee.devicesIterator(utils.deviceNotCoordinator)) {
             if (utils.isAvailabilityEnabledForEntity(device, settings.get())) {
                 this.resetTimer(device);
 
@@ -153,7 +153,7 @@ export default class Availability extends Extension {
     }
 
     @bind private async publishAvailabilityForAllEntities(): Promise<void> {
-        for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+        for (const entity of this.zigbee.devicesAndGroupsIterator(utils.deviceNotCoordinator)) {
             if (utils.isAvailabilityEnabledForEntity(entity, settings.get())) {
                 await this.publishAvailability(entity, true, false, true);
             }
@@ -187,7 +187,7 @@ export default class Availability extends Extension {
         await this.mqtt.publish(topic, payload, {retain: true, qos: 1});
 
         if (!skipGroups && entity.isDevice()) {
-            for (const group of this.zigbee.groups()) {
+            for (const group of this.zigbee.groupsIterator()) {
                 if (group.hasMember(entity) && utils.isAvailabilityEnabledForEntity(group, settings.get())) {
                     await this.publishAvailability(group, false, forcePublish);
                 }

lib/extension/bind.ts

@@ -363,7 +363,7 @@ export default class Bind extends Extension {
         if (data.action === 'add') {
             const bindsToGroup: zh.Bind[] = [];
 
-            for (const device of this.zigbee.devices(false)) {
+            for (const device of this.zigbee.devicesIterator(utils.deviceNotCoordinator)) {
                 for (const endpoint of device.zh.endpoints) {
                     for (const bind of endpoint.binds) {
                         if (bind.target === data.group.zh) {
@@ -443,7 +443,7 @@ export default class Bind extends Extension {
         const endpoints = utils.isEndpoint(target) ? [target] : target.members;
         const allBinds: zh.Bind[] = [];
 
-        for (const device of this.zigbee.devices(false)) {
+        for (const device of this.zigbee.devicesIterator(utils.deviceNotCoordinator)) {
             for (const endpoint of device.zh.endpoints) {
                 for (const bind of endpoint.binds) {
                     allBinds.push(bind);

lib/extension/bridge.ts

@@ -725,8 +725,12 @@ export default class Bridge extends Extension {
             scenes: Scene[];
         }
 
-        const devices = this.zigbee.devices().map((device) => {
+        // XXX: definition<>DefinitionPayload don't match to use `Device[]` type here
+        const devices: KeyValue[] = [];
+
+        for (const device of this.zigbee.devicesIterator()) {
             const endpoints: {[s: number]: Data} = {};
+
             for (const endpoint of device.zh.endpoints) {
                 const data: Data = {
                     scenes: utils.getScenes(endpoint),
@@ -758,7 +762,7 @@ export default class Bridge extends Extension {
                 endpoints[endpoint.ID] = data;
             }
 
-            return {
+            devices.push({
                 ieee_address: device.ieeeAddr,
                 type: device.zh.type,
                 network_address: device.zh.networkAddress,
@@ -775,24 +779,32 @@ export default class Bridge extends Extension {
                 interview_completed: device.zh.interviewCompleted,
                 manufacturer: device.zh.manufacturerName,
                 endpoints,
-            };
-        });
+            });
+        }
 
         await this.mqtt.publish('bridge/devices', stringify(devices), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
     async publishGroups(): Promise<void> {
-        const groups = this.zigbee.groups().map((g) => {
-            return {
-                id: g.ID,
-                friendly_name: g.ID === 901 ? 'default_bind_group' : g.name,
-                description: g.options.description,
-                scenes: utils.getScenes(g.zh),
-                members: g.zh.members.map((e) => {
-                    return {ieee_address: e.getDevice().ieeeAddr, endpoint: e.ID};
-                }),
-            };
-        });
+        // XXX: id<>ID can't use `Group[]` type
+        const groups: KeyValue[] = [];
+
+        for (const group of this.zigbee.groupsIterator()) {
+            const members = [];
+
+            for (const member of group.zh.members) {
+                members.push({ieee_address: member.getDevice().ieeeAddr, endpoint: member.ID});
+            }
+
+            groups.push({
+                id: group.ID,
+                friendly_name: group.ID === 901 ? 'default_bind_group' : group.name,
+                description: group.options.description,
+                scenes: utils.getScenes(group.zh),
+                members,
+            });
+        }
+
         await this.mqtt.publish('bridge/groups', stringify(groups), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
@@ -807,20 +819,23 @@ export default class Bridge extends Extension {
             custom_clusters: {},
         };
 
-        for (const device of this.zigbee.devices()) {
-            if (Object.keys(device.customClusters).length !== 0) {
-                data.custom_clusters[device.ieeeAddr] = device.customClusters;
-            }
+        for (const device of this.zigbee.devicesIterator((d) => !utils.objectIsEmpty(d.customClusters))) {
+            data.custom_clusters[device.ieeeAddr] = device.customClusters;
         }
 
         await this.mqtt.publish('bridge/definitions', stringify(data), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
-    getDefinitionPayload(device: Device): DefinitionPayload {
-        if (!device.definition) return null;
+    getDefinitionPayload(device: Device): DefinitionPayload | null {
+        if (!device.definition) {
+            return null;
+        }
+
+        // TODO: better typing to avoid @ts-expect-error
         // @ts-expect-error icon is valid for external definitions
         const definitionIcon = device.definition.icon;
         let icon = device.options.icon ?? definitionIcon;
+
         if (icon) {
             icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(device.zh.modelID));
             icon = icon.replace('${model}', utils.sanitizeImageParameter(device.definition.model));

lib/extension/configure.ts

@@ -68,9 +68,8 @@ export default class Configure extends Extension {
         setImmediate(async () => {
             // Only configure routers on startup, end devices are likely sleeping and
             // will reconfigure once they send a message
-            for (const device of this.zigbee.devices(false).filter((d) => d.zh.type === 'Router')) {
-                // Sleep 10 seconds between configuring on startup to not DDoS the coordinator
-                // when many devices have to be configured.
+            for (const device of this.zigbee.devicesIterator((d) => d.type === 'Router')) {
+                // Sleep 10 seconds between configuring on startup to not DDoS the coordinator when many devices have to be configured.
                 await utils.sleep(10);
                 await this.configure(device, 'started');
             }

lib/extension/frontend.ts

@@ -139,7 +139,7 @@ export default class Frontend extends Extension {
             }
         }
 
-        for (const device of this.zigbee.devices(false)) {
+        for (const device of this.zigbee.devicesIterator(utils.deviceNotCoordinator)) {
             const payload = this.state.get(device);
             const lastSeen = settings.get().advanced.last_seen;
             /* istanbul ignore if */

lib/extension/groups.ts

@@ -51,7 +51,6 @@ export default class Groups extends Extension {
 
     private async syncGroupsWithSettings(): Promise<void> {
         const settingsGroups = settings.getGroups();
-        const zigbeeGroups = this.zigbee.groups();
 
         const addRemoveFromGroup = async (
             action: 'add' | 'remove',
@@ -76,7 +75,7 @@ export default class Groups extends Extension {
 
         for (const settingGroup of settingsGroups) {
             const groupID = settingGroup.ID;
-            const zigbeeGroup = zigbeeGroups.find((g) => g.ID === groupID) || this.zigbee.createGroup(groupID);
+            const zigbeeGroup = this.zigbee.groupsIterator((g) => g.groupID === groupID).next().value || this.zigbee.createGroup(groupID);
             const settingsEndpoints: zh.Endpoint[] = [];
 
             for (const d of settingGroup.devices) {
@@ -113,13 +112,11 @@ export default class Groups extends Extension {
             }
         }
 
-        for (const zigbeeGroup of zigbeeGroups) {
-            if (!settingsGroups.some((g) => g.ID === zigbeeGroup.ID)) {
-                for (const endpoint of zigbeeGroup.zh.members) {
-                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendly_name;
+        for (const zigbeeGroup of this.zigbee.groupsIterator((zg) => !settingsGroups.some((sg) => sg.ID === zg.groupID))) {
+            for (const endpoint of zigbeeGroup.zh.members) {
+                const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendly_name;
 
-                    await addRemoveFromGroup('remove', deviceName, zigbeeGroup.ID, endpoint, zigbeeGroup);
-                }
+                await addRemoveFromGroup('remove', deviceName, zigbeeGroup.ID, endpoint, zigbeeGroup);
             }
         }
     }
@@ -153,7 +150,13 @@ export default class Groups extends Extension {
 
         if (payloadKeys.length) {
             const entity = data.entity;
-            const groups = this.zigbee.groups().filter((g) => g.options && (g.options.optimistic == undefined || g.options.optimistic));
+            const groups = [];
+
+            for (const group of this.zigbee.groupsIterator()) {
+                if (group.options && (group.options.optimistic == undefined || group.options.optimistic)) {
+                    groups.push(group);
+                }
+            }
 
             if (entity instanceof Device) {
                 for (const group of groups) {
@@ -359,7 +362,7 @@ export default class Groups extends Extension {
             resolvedEntityEndpoint,
         } = parsed;
         let error = parsed.error;
-        let changedGroups: Group[] = [];
+        const changedGroups: Group[] = [];
 
         if (!error) {
             try {
@@ -406,7 +409,11 @@ export default class Groups extends Extension {
                 } else {
                     // remove_all
                     logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
-                    changedGroups = this.zigbee.groups().filter((g) => g.zh.members.includes(resolvedEntityEndpoint));
+
+                    for (const group of this.zigbee.groupsIterator((g) => g.members.includes(resolvedEntityEndpoint))) {
+                        changedGroups.push(group);
+                    }
+
                     await resolvedEntityEndpoint.removeFromAllGroups();
 
                     for (const settingsGroup of settings.getGroups()) {

lib/extension/homeassistant.ts

@@ -225,7 +225,9 @@ export default class HomeAssistant extends Extension {
         const discoverWait = 5;
         // Discover with `published = false`, this will populate `this.discovered` without publishing the discoveries.
         // This is needed for clearing outdated entries in `this.onMQTTMessage()`
-        for (const e of [this.bridge, ...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+        await this.discover(this.bridge, false);
+
+        for (const e of this.zigbee.devicesAndGroupsIterator(utils.deviceNotCoordinator)) {
             await this.discover(e, false);
         }
 
@@ -235,7 +237,9 @@ export default class HomeAssistant extends Extension {
             this.mqtt.unsubscribe(`${this.discoveryTopic}/#`);
             logger.debug(`Discovering entities to Home Assistant`);
 
-            for (const e of [this.bridge, ...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+            await this.discover(this.bridge);
+
+            for (const e of this.zigbee.devicesAndGroupsIterator(utils.deviceNotCoordinator)) {
                 await this.discover(e);
             }
         }, utils.seconds(discoverWait));
@@ -1780,7 +1784,7 @@ export default class HomeAssistant extends Extension {
         } else if ((data.topic === this.statusTopic || data.topic === defaultStatusTopic) && data.message.toLowerCase() === 'online') {
             const timer = setTimeout(async () => {
                 // Publish all device states.
-                for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+                for (const entity of this.zigbee.devicesAndGroupsIterator(utils.deviceNotCoordinator)) {
                     if (this.state.exists(entity)) {
                         await this.publishEntityState(entity, this.state.get(entity), 'publishCached');
                     }

lib/extension/legacy/bridgeLegacy.ts

@@ -127,7 +127,9 @@ export default class BridgeLegacy extends Extension {
 
     @bind async devices(topic: string): Promise<void> {
         const coordinator = await this.zigbee.getCoordinatorVersion();
-        const devices = this.zigbee.devices().map((device) => {
+        const devices: KeyValue[] = [];
+
+        for (const device of this.zigbee.devicesIterator()) {
             const payload: KeyValue = {
                 ieeeAddr: device.ieeeAddr,
                 type: device.zh.type,
@@ -155,8 +157,8 @@ export default class BridgeLegacy extends Extension {
                 payload.lastSeen = Date.now();
             }
 
-            return payload;
-        });
+            devices.push(payload);
+        }
 
         if (topic.split('/').pop() == 'get') {
             await this.mqtt.publish(`bridge/config/devices`, stringify(devices), {}, settings.get().mqtt.base_topic, false, false);

lib/extension/legacy/report.ts

@@ -2,6 +2,7 @@ import * as zhc from 'zigbee-herdsman-converters';
 
 import logger from '../../util/logger';
 import * as settings from '../../util/settings';
+import utils from '../../util/utils';
 import Extension from '../extension';
 
 const defaultConfiguration = {
@@ -178,7 +179,7 @@ export default class Report extends Extension {
     }
 
     override async start(): Promise<void> {
-        for (const device of this.zigbee.devices(false)) {
+        for (const device of this.zigbee.devicesIterator(utils.deviceNotCoordinator)) {
             if (this.shouldSetupReporting(device, null)) {
                 await this.setupReporting(device);
             }