Some headers and metadata not included in the logs sent by log_hooks:new_json

I’ve been receiving json webhook calls using this code, and added the direction meta and Message-Id header to the log_parameters today and restarted the kumomta daemon, but I’m still only getting Subject, From, customer_id and domain_id in the JSON payload. Am I doing something wrong?

--
local LOG_WEBHOOK_URL = "http://10.1.0.4:4242/log"
local MSG_WEBHOOK_URL = "http://10.1.0.4:4242/message"
local TENANT_CAN_SEND_URL = "http://10.1.0.4:4243/v1/account/%s/can-send"
--

local kumo = require 'kumo'
local sources = require 'policy-extras.sources'
local dkim_sign = require 'dkim_sign'
local log_hooks = require 'policy-extras.log_hooks'
local shaping = require 'policy-extras.shaping'
local sqlite = require 'sqlite'

-- Custom queue helper
local queue_module = require 'queue'

local listener_domains = require 'policy-extras.listener_domains'

local dkim_signer = dkim_sign:setup { '/opt/kumomta/etc/policy/config/dkim.toml' }

local shaper = shaping:setup_with_automation {
  publish = { 'http://127.0.0.1:8008' },
  subscribe = { 'http://127.0.0.1:8008' },
  extra_files = { '/opt/kumomta/etc/policy/config/shaping.toml', '/opt/kumomta/etc/policy/config/shaping-custom.toml' },
}


log_hooks:new_json {
  name = "webhook",
  -- log_parameters are combined with the name and
  -- passed through to kumo.configure_log_hook
  log_parameters = {
    headers = { 'Subject', 'From', 'Message-Id' },
    meta = { 'tenant', 'domain_id', 'customer_id', 'direction' }
  },
  -- queue config are passed to kumo.make_queue_config.
  -- You can use these to override the retry parameters
  -- if you wish.
  -- The defaults are shown below.
  queue_config = {
    retry_interval = "1m",
    max_retry_interval = "20m",
  },
  -- The URL to POST the JSON to
  url = LOG_WEBHOOK_URL
}

--- the rest of init.lua

Hey there @original-baboon, thanks for posting. Please read the “Troubleshooting” and “How to Ask for Help” buttons below. If you would like a 1:1 support session from the KumoMTA team, details are at the “Book a Support Session” button below.

Sample json payload received by my webhook handler:

    {
        "type": "Reception",
        "id": "79c6b4e6d40511ee98d8960002cafe7c",
        "sender": "info@kumo.example.com",
        "recipient": "recipient@gmail.com",
        "queue": "4cdd7bdd-294e-4762-892f-83d40abf5a87@gmail.com",
        "site": "",
        "size": 51652,
        "response": {
            "code": 250,
            "enhanced_code": null,
            "content": "",
            "command": ""
        },
        "peer_address": {
            "name": "example.com",
            "addr": "1.2.3.4"
        },
        "timestamp": 1708883047,
        "created": 1708883047,
        "num_attempts": 0,
        "bounce_classification": "Uncategorized",
        "egress_pool": "",
        "egress_source": "",
        "feedback_report": "",
        "meta": {
            "domain_id": "be6f46e9-089c-47c3-92b1-376a061924ba",
            "customer_id": "4cdd7bdd-294e-4762-892f-83d40abf5a87"
        },
        "headers": {
            "Subject": "this is the subject",
            "From": "Sender \u003cinfo@kumo.example.com\u003e"
        },
        "delivery_protocol": "",
        "reception_protocol": "ESMTP",
        "nodeid": "2f6489ba-9d9d-47d5-83bb-8492ac3fdd93"
    }
    {
        "type": "Delivery",
        "id": "79c6b4e6d40511ee98d8960002cafe7c",
        "sender": "info@kumo.example.com",
        "recipient": "recipient@gmail.com",
        "queue": "4cdd7bdd-294e-4762-892f-83d40abf5a87@gmail.com",
        "site": "srv2-\u003e(alt1|alt2|alt3|alt4)?.gmail-smtp-in.l.google.com@smtp_client",
        "size": 51652,
        "response": {
            "code": 250,
            "enhanced_code": {
                "class": 2,
                "subject": 0,
                "detail": 0
            },
            "content": "OK  1708883048 e5-20020aa78c45000000b006e3fce37039si2397734pfd.351 - gsmtp",
            "command": ".\r\n"
        },
        "peer_address": {
            "name": "gmail-smtp-in.l.google.com.",
            "addr": "74.125.20.26"
        },
        "timestamp": 1708883048,
        "created": 1708883047,
        "num_attempts": 0,
        "bounce_classification": "Uncategorized",
        "egress_pool": "default-pool",
        "egress_source": "srv2",
        "feedback_report": "",
        "meta": {
            "domain_id": "be6f46e9-089c-47c3-92b1-376a061924ba",
            "customer_id": "4cdd7bdd-294e-4762-892f-83d40abf5a87"
        },
        "headers": {
            "Subject": "this is the subject",
            "From": "Sender \u003cinfo@kumo.example.com\u003e"
        },
        "delivery_protocol": "ESMTP",
        "reception_protocol": "ESMTP",
        "nodeid": "2f6489ba-9d9d-47d5-83bb-8492ac3fdd93"
    }

headers of the same message

Never mind, the issue was in my own code outside Kumo :man_facepalming:

It happens, thanks for the update.