ERR_OPS_APPLY_CHANGE_SET when mandatory Custom Field exists

[lukassavukaitis]

Diode version

v1.0.0

Deployment type

docker compose

Deployment type version

Docker Compose version v2.35.1

Diode SDK type

diode-sdk-python

Diode SDK version

Orb agent v2.0.0

Diode NetBox Plugin version

v1.0.0

NetBox version

v4.2.6

Steps to reproduce

Create a Custom Field associated to dcim.device model and make it required.
Create a device manually.
Create a device discovery in Orb agent to discover the same device.

Expected behavior

Device is discovered and any missing pieces are added into Netbox.

Observed behavior

Device discovery fails with "error":"apply change set failed - ERR_OPS_APPLY_CHANGE_SET - {\"id\":\"2a886e79-23ff-44c7-8319-3349af2ba319\",\"errors\":{\"dcim.device\":{\"__all__\":[\"Missing required custom field 'Ownership'.\"

Excerpt from reconciler container log:

{
    "time": "2025-05-05T09:40:25.83555814Z",
    "level": "DEBUG",
    "msg": "generate diff",
    "statusCode": 200,
    "response": {
        "id": "2a886e79-23ff-44c7-8319-3349af2ba319",
        "errors": null,
        "change_set": {
            "id": "2a886e79-23ff-44c7-8319-3349af2ba319",
            "changes": [{
                    "id": "b0125ae1-29d3-41e6-a48b-da0f093d6f6b",
                    "change_type": "noop",
                    "object_type": "dcim.devicerole",
                    "object_id": 2,
                    "ref_id": null,
                    "object_primary_value": "LAN Switch",
                    "before": {
                        "color": "ffeb3b",
                        "description": "Standard LAN-Switch Role for all belongings (shared and dedicated LAN)",
                        "id": 2,
                        "name": "LAN Switch",
                        "slug": "lan-switch",
                        "vm_role": false
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "34ebbcf8-3911-46cd-9e44-534b29228f51",
                    "change_type": "noop",
                    "object_type": "dcim.site",
                    "object_id": 23,
                    "ref_id": null,
                    "object_primary_value": "My Site",
                    "before": {
                        "description": "My Site office",
                        "group": 24,
                        "id": 23,
                        "latitude": 54.123456,
                        "longitude": 23.123456,
                        "name": "My Site",
                        "physical_address": "1600 Pennsylvania Avenue NW",
                        "region": 4,
                        "shipping_address": "1600 Pennsylvania Avenue NW",
                        "slug": "site-my-site",
                        "status": "active",
                        "tenant": 14
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "e945da26-6b9d-4718-b7f6-5937c6ba490f",
                    "change_type": "noop",
                    "object_type": "dcim.manufacturer",
                    "object_id": 1,
                    "ref_id": null,
                    "object_primary_value": "Cisco",
                    "before": {
                        "custom_fields": {},
                        "id": 1,
                        "name": "Cisco",
                        "slug": "cisco"
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "f4832b28-9dc4-4215-a5a9-bf0cf46c61e7",
                    "change_type": "noop",
                    "object_type": "dcim.devicetype",
                    "object_id": 1973,
                    "ref_id": null,
                    "object_primary_value": "WS-C2960X-48FPD-L",
                    "before": {
                        "custom_fields": {},
                        "exclude_from_utilization": false,
                        "id": 1973,
                        "is_full_depth": false,
                        "manufacturer": 1,
                        "model": "WS-C2960X-48FPD-L",
                        "part_number": "WS-C2960X-48FPD-L",
                        "slug": "ws-c2960x-48fpd-l",
                        "u_height": 1.0
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "4020f58d-e2cd-47ab-8724-99d5b289fc6b",
                    "change_type": "update",
                    "object_type": "dcim.platform",
                    "object_id": 368,
                    "ref_id": null,
                    "object_primary_value": "ios",
                    "before": {
                        "id": 368,
                        "name": "ios",
                        "slug": "ios"
                    },
                    "data": {
                        "id": 368,
                        "manufacturer": 1,
                        "name": "ios"
                    },
                    "new_refs": []
                }, {
                    "id": "ae44d4d5-92e7-400d-907e-6b6722a7a78d",
                    "change_type": "create",
                    "object_type": "dcim.device",
                    "object_id": null,
                    "ref_id": "new_object:dcim.device:1b53fedb-81a1-4d33-aebe-ed29156ea069",
                    "object_primary_value": "My-Switch-Name",
                    "before": {},
                    "data": {
                        "custom_fields": {},
                        "device_type": 1973,
                        "name": "My-Switch-Name",
                        "platform": 368,
                        "role": 2,
                        "serial": "FOC2866S1G8",
                        "site": 23,
                        "status": "active"
                    },
                    "new_refs": []
                }
            "branch": null
        }
    }
}{
    "time": "2025-05-05T09:40:25.84171016Z",
    "level": "DEBUG",
    "msg": "change set generated",
    "id": 508,
    "externalID": "2a886e79-23ff-44c7-8319-3349af2ba319",
    "ingestionLogID": 508
}{
    "time": "2025-05-05T09:40:25.918239048Z",
    "level": "DEBUG",
    "msg": "apply change set",
    "payload": {
        "id": "2a886e79-23ff-44c7-8319-3349af2ba319",
        "changes": [{
                "id": "b0125ae1-29d3-41e6-a48b-da0f093d6f6b",
                "change_type": "noop",
                "object_type": "dcim.devicerole",
                "object_id": 2,
                "data": {},
                "object_primary_value": "LAN Switch"
            }, {
                "id": "34ebbcf8-3911-46cd-9e44-534b29228f51",
                "change_type": "noop",
                "object_type": "dcim.site",
                "object_id": 23,
                "data": {},
                "object_primary_value": "My Site"
            }, {
                "id": "e945da26-6b9d-4718-b7f6-5937c6ba490f",
                "change_type": "noop",
                "object_type": "dcim.manufacturer",
                "object_id": 1,
                "data": {},
                "object_primary_value": "Cisco"
            }, {
                "id": "f4832b28-9dc4-4215-a5a9-bf0cf46c61e7",
                "change_type": "noop",
                "object_type": "dcim.devicetype",
                "object_id": 1973,
                "data": {},
                "object_primary_value": "WS-C2960X-48FPD-L"
            }, {
                "id": "4020f58d-e2cd-47ab-8724-99d5b289fc6b",
                "change_type": "update",
                "object_type": "dcim.platform",
                "object_id": 368,
                "data": {
                    "id": 368,
                    "manufacturer": 1,
                    "name": "ios"
                },
                "object_primary_value": "ios"
            }, {
                "id": "ae44d4d5-92e7-400d-907e-6b6722a7a78d",
                "change_type": "create",
                "object_type": "dcim.device",
                "ref_id": "new_object:dcim.device:1b53fedb-81a1-4d33-aebe-ed29156ea069",
                "data": {
                    "custom_fields": {},
                    "device_type": 1973,
                    "name": "My-Switch-Name",
                    "platform": 368,
                    "role": 2,
                    "serial": "FOC2866S1G8",
                    "site": 23,
                    "status": "active"
                },
                "object_primary_value": "My-Switch-Name"
            }
        ]
    }
}{
    "time": "2025-05-05T09:40:25.95994901Z",
    "level": "DEBUG",
    "msg": "apply change set",
    "response": {
        "id": "2a886e79-23ff-44c7-8319-3349af2ba319",
        "errors": {
            "dcim.device": {
                "__all__": ["Missing required custom field 'Ownership'."]
            }
        }
    }
}{
    "time": "2025-05-05T09:40:25.959994177Z",
    "level": "DEBUG",
    "msg": "failed to apply change set",
    "id": 508,
    "externalID": "2a886e79-23ff-44c7-8319-3349af2ba319",
    "ingestionLogID": 508,
    "error": "apply change set failed - ERR_OPS_APPLY_CHANGE_SET - {"id":"2a886e79-23ff-44c7-8319-3349af2ba319","errors":{"dcim.device":{"__all__":["Missing required custom field 'Ownership'."]}}}"
}{
    "time": "2025-05-05T09:40:25.962210968Z",
    "level": "ERROR",
    "msg": "error applying changeset",
    "error": "apply change set failed - ERR_OPS_APPLY_CHANGE_SET - {"id":"2a886e79-23ff-44c7-8319-3349af2ba319","errors":{"dcim.device":{"__all__":["Missing required custom field 'Ownership'."]}}}"
}

[ltucker]

If I understand correctly, the expectation is that we would match an existing device which already has the required field supplied and update it. The observed behavior is an attempt to create a new device which fails because it does not have the required field supplied (expected behavior for a new device).

Thanks for the detailed log capture. It would also be helpful to have a more complete description of the object that was created in Step 2 above. — a screen cap of the NetBox page for it would be fine.

[lukassavukaitis]

@ltucker yes, you've understood the problem right.
I have attached a couple of screenshots on original post, hope this helps.

[ltucker]

Thanks! In this case, I believe the matching is being (overly) strict and considering this a separate device because there is a tenant specified on the existing device and none on the discovered device.

I believe the fix will be to loosen this up in diode's matching and have it consider these the same (so long as there is not an ambiguity with another device with that name/site in another tenant or lacking a tenant).

It appears to me that orb-discovery policies don't currently allow specifying a default tenant on a device, that would also provide potential help perhaps -- I filed netboxlabs/orb-agent#104 over there as well to track.

[lukassavukaitis]

So I've seen netboxlabs/orb-agent#104 was closed however I'm now hitting another issue - it now tries to create a Tenant which I specified in device discovery policy (and it fails, since I have another mandatory custom field for Tenant)

[ltucker]

It should be able to match an existing Tenant just by name if it has been created on the NetBox side with required fields -- does it exist there and it's not finding it or it is in fact a new Tenant?

If we want it to create the Tenant in discovery with required custom fields, orb would need to support a richer way of describing the defaults with more fields in the policy I think. (Diode's ingest API should already support specifying custom fields) -- alternatively (though this may not make sense in this case) -- I believe a default value for the custom field would be honored as well.

[lukassavukaitis]

Tenant with the same name exists in Netbox, it just seems it tries to create a new one. Looking at the log entry, slug is different though of what we have in Netbox.
Not sure if that's also the reason why it tries to create a device too instead of just updating it.

{
    "time": "2025-05-08T13:09:36.013321338Z",
    "level": "DEBUG",
    "msg": "generate diff",
    "statusCode": 200,
    "response": {
        "id": "91fb1a0a-e9c3-4123-aaf4-54af49d0c8da",
        "errors": null,
        "change_set": {
            "id": "91fb1a0a-e9c3-4123-aaf4-54af49d0c8da",
            "changes": [{
                    "id": "9f150461-533c-4a95-90e8-35c5a2506962",
                    "change_type": "noop",
                    "object_type": "dcim.devicerole",
                    "object_id": 2,
                    "ref_id": null,
                    "object_primary_value": "LAN Switch",
                    "before": {
                        "color": "ffeb3b",
                        "description": "Standard LAN-Switch Role for all belongings (shared and dedicated LAN)",
                        "id": 2,
                        "name": "LAN Switch",
                        "slug": "lan-switch",
                        "vm_role": false
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "f529443c-550f-45cc-a1df-305e5229db0d",
                    "change_type": "noop",
                    "object_type": "dcim.site",
                    "object_id": 23,
                    "ref_id": null,
                    "object_primary_value": "My Site",
                    "before": {
                        "description": "My Site office",
                        "group": 24,
                        "id": 23,
                        "latitude": 54.123456,
                        "longitude": 23.123456,
                        "name": "My Site",
                        "physical_address": "1600 Pennsylvania Avenue NW",
                        "region": 4,
                        "shipping_address": "1600 Pennsylvania Avenue NW",
                        "slug": "site-my-site",
                        "status": "active",
                        "tenant": 14
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "c1291704-28ff-432b-9280-7c3f5794df75",
                    "change_type": "create",
                    "object_type": "tenancy.tenant",
                    "object_id": null,
                    "ref_id": "new_object:tenancy.tenant:bae3f110-4b80-49fa-ade5-676cd4397275",
                    "object_primary_value": "My Tenant",
                    "before": {},
                    "data": {
                        "custom_fields": {},
                        "name": "My Tenant",
                        "slug": "my-tenant"
                    },
                    "new_refs": []
                }, {
                    "id": "4e686a3a-7035-42b5-b5a1-d90383079880",
                    "change_type": "noop",
                    "object_type": "dcim.manufacturer",
                    "object_id": 1,
                    "ref_id": null,
                    "object_primary_value": "Cisco",
                    "before": {
                        "custom_fields": {},
                        "id": 1,
                        "name": "Cisco",
                        "slug": "cisco"
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "836eaf3a-8ff1-4548-be6f-fc6d9530fd4b",
                    "change_type": "noop",
                    "object_type": "dcim.location",
                    "object_id": 351,
                    "ref_id": null,
                    "object_primary_value": "My Location",
                    "before": {
                        "id": 351,
                        "name": "My Location",
                        "site": 23,
                        "slug": "my-location",
                        "status": "active",
                        "tenant": 86
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "30203f3d-a2b1-4e2f-93d1-b4fb1041f18e",
                    "change_type": "noop",
                    "object_type": "dcim.devicetype",
                    "object_id": 1973,
                    "ref_id": null,
                    "object_primary_value": "WS-C2960X-48FPD-L",
                    "before": {
                        "custom_fields": {},
                        "exclude_from_utilization": false,
                        "id": 1973,
                        "is_full_depth": false,
                        "manufacturer": 1,
                        "model": "WS-C2960X-48FPD-L",
                        "part_number": "WS-C2960X-48FPD-L",
                        "slug": "ws-c2960x-48fpd-l",
                        "u_height": 1.0
                    },
                    "data": {},
                    "new_refs": []
                }, {
                    "id": "ca333866-e080-4c84-afe6-cb755084c9ee",
                    "change_type": "update",
                    "object_type": "dcim.platform",
                    "object_id": 368,
                    "ref_id": null,
                    "object_primary_value": "ios",
                    "before": {
                        "id": 368,
                        "name": "ios",
                        "slug": "ios"
                    },
                    "data": {
                        "id": 368,
                        "manufacturer": 1,
                        "name": "ios"
                    },
                    "new_refs": []
                }, {
                    "id": "3ea45884-9af3-4465-922b-13b902ff5d23",
                    "change_type": "create",
                    "object_type": "dcim.device",
                    "object_id": null,
                    "ref_id": "new_object:dcim.device:90245a30-462b-456a-9c9b-1ca0f4539307",
                    "object_primary_value": "My-Switch-Name",
                    "before": {},
                    "data": {
                        "custom_fields": {},
                        "device_type": 1973,
                        "location": 351,
                        "name": "My-Switch-Name",
                        "platform": 368,
                        "role": 2,
                        "serial": "FOC2866S1G8",
                        "site": 23,
                        "status": "active",
                        "tenant": "new_object:tenancy.tenant:bae3f110-4b80-49fa-ade5-676cd4397275"
                    },
                    "new_refs": ["tenant"]
                }
            ],
            "branch": null
        }
    }
}{
    "time": "2025-05-08T13:09:36.032926746Z",
    "level": "DEBUG",
    "msg": "change set generated",
    "id": 667,
    "externalID": "91fb1a0a-e9c3-4123-aaf4-54af49d0c8da",
    "ingestionLogID": 667
}{
    "time": "2025-05-08T13:09:36.033146031Z",
    "level": "DEBUG",
    "msg": "apply change set",
    "payload": {
        "id": "91fb1a0a-e9c3-4123-aaf4-54af49d0c8da",
        "changes": [{
                "id": "9f150461-533c-4a95-90e8-35c5a2506962",
                "change_type": "noop",
                "object_type": "dcim.devicerole",
                "object_id": 2,
                "data": {},
                "object_primary_value": "LAN Switch"
            }, {
                "id": "f529443c-550f-45cc-a1df-305e5229db0d",
                "change_type": "noop",
                "object_type": "dcim.site",
                "object_id": 23,
                "data": {},
                "object_primary_value": "My Site"
            }, {
                "id": "c1291704-28ff-432b-9280-7c3f5794df75",
                "change_type": "create",
                "object_type": "tenancy.tenant",
                "ref_id": "new_object:tenancy.tenant:bae3f110-4b80-49fa-ade5-676cd4397275",
                "data": {
                    "custom_fields": {},
                    "name": "My Tenant",
                    "slug": "my-tenant"
                },
                "object_primary_value": "My Tenant"
            }, {
                "id": "4e686a3a-7035-42b5-b5a1-d90383079880",
                "change_type": "noop",
                "object_type": "dcim.manufacturer",
                "object_id": 1,
                "data": {},
                "object_primary_value": "Cisco"
            }, {
                "id": "836eaf3a-8ff1-4548-be6f-fc6d9530fd4b",
                "change_type": "noop",
                "object_type": "dcim.location",
                "object_id": 351,
                "data": {},
                "object_primary_value": "My Location"
            }, {
                "id": "30203f3d-a2b1-4e2f-93d1-b4fb1041f18e",
                "change_type": "noop",
                "object_type": "dcim.devicetype",
                "object_id": 1973,
                "data": {},
                "object_primary_value": "WS-C2960X-48FPD-L"
            }, {
                "id": "ca333866-e080-4c84-afe6-cb755084c9ee",
                "change_type": "update",
                "object_type": "dcim.platform",
                "object_id": 368,
                "data": {
                    "id": 368,
                    "manufacturer": 1,
                    "name": "ios"
                },
                "object_primary_value": "ios"
            }, {
                "id": "3ea45884-9af3-4465-922b-13b902ff5d23",
                "change_type": "create",
                "object_type": "dcim.device",
                "ref_id": "new_object:dcim.device:90245a30-462b-456a-9c9b-1ca0f4539307",
                "data": {
                    "custom_fields": {},
                    "device_type": 1973,
                    "location": 351,
                    "name": "My-Switch-Name",
                    "platform": 368,
                    "role": 2,
                    "serial": "FOC2866S1G8",
                    "site": 23,
                    "status": "active",
                    "tenant": "new_object:tenancy.tenant:bae3f110-4b80-49fa-ade5-676cd4397275"
                },
                "object_primary_value": "My-Switch-Name"",
                "new_refs": ["tenant"]
            }
        ]
    }
}{
    "time": "2025-05-08T13:09:36.07169557Z",
    "level": "DEBUG",
    "msg": "apply change set",
    "response": "{"id":"91fb1a0a-e9c3-4123-aaf4-54af49d0c8da","errors":{"tenancy.tenant":{"__all__":["Missing required custom field 'CRM_ID'."]}}}"
}{
    "time": "2025-05-08T13:09:36.071749767Z",
    "level": "DEBUG",
    "msg": "failed to apply change set",
    "id": 667,
    "externalID": "91fb1a0a-e9c3-4123-aaf4-54af49d0c8da",
    "ingestionLogID": 667,
    "error": "apply change set failed - ERR_OPS_APPLY_CHANGE_SET - {"id":"91fb1a0a-e9c3-4123-aaf4-54af49d0c8da","errors":{"tenancy.tenant":{"__all__":["Missing required custom field 'CRM_ID'."]}}}"
}{
    "time": "2025-05-08T13:09:36.079715542Z",
    "level": "ERROR",
    "msg": "error applying changeset",
    "error": "apply change set failed - ERR_OPS_APPLY_CHANGE_SET - {"id":"91fb1a0a-e9c3-4123-aaf4-54af49d0c8da","errors":{"tenancy.tenant":{"__all__":["Missing required custom field 'CRM_ID'."]}}}"
}

[ltucker]

Possibly a similar situation of overly strict diode, does the Tenant in NetBox have a TenantGroup ?

[lukassavukaitis]

Yes, it does.

[sepetuks]

there is another case in orb agent project that I think is related (partialy) to this conversation
netboxlabs/orb-agent#81

p.s. adding just to have a linkage