var/opt/nydus/ops/primordial/log/envelope.py000064400000015417147205553620015234 0ustar00# -*- coding: utf-8 -*- import abc import logging import logging.config from datetime import datetime from typing import Optional # pylint: disable=W0611 import pytz from primordial.timeutils import iso8601_utc from primordial.log.extra import LogExtra from primordial.log.data import LogData from primordial.log.envelopedata import EnvelopeVersion, EnvelopeRole, ConfigLogData, HostLogData, PayloadLogData LOG = logging.getLogger(__name__) # Abstract Factory class class Envelope(LogData, metaclass=abc.ABCMeta): """Customized envelope for sending log data. Given a log record, envelope version and configuration data, constructs a log message in the required format which can be returned as a json. """ ENVELOPE_VERSION_FIELD = 'envelope_version' DATA_SCHEMA_FIELD = 'data_schema' @staticmethod def getEnvelope(record: logging.LogRecord, configLogData: ConfigLogData, envelopeVersion: Optional[EnvelopeVersion] = None) -> 'Envelope': """A factory method for producing an Envelope instance. :param record: The log record :param configLogData: The logging data :param envelopeVersion: The version of the envelope :returns: A new Envelope object :rtype: :class:`Envelope` """ extra = getattr(record, LogExtra.EXTRA_FIELD, {}) logextra = None if isinstance(extra, dict): try: logextra = LogExtra.getLogExtra(**extra) except Exception as e: # pylint: disable=broad-except LOG.error("Unable to process extra log fields: %s", str(e)) if not isinstance(envelopeVersion, EnvelopeVersion): envelopeVersion = EnvelopeVersion() if isinstance(logextra, LogExtra): if logextra.get(Envelope.ENVELOPE_VERSION_FIELD) is not None: version = logextra.get(Envelope.ENVELOPE_VERSION_FIELD) envelopeVersion = EnvelopeVersion.fromString(version) major = envelopeVersion.get(EnvelopeVersion.MAJOR_FIELD) # type: ignore if major == 1: return EnvelopeV1(record, envelopeVersion, configLogData, logextra) # type: ignore LOG.error("Unknown log envelope major version %s. Using default version", str(major)) return Envelope.getEnvelope(record, configLogData, EnvelopeVersion()) # Envelope v1. Extend and overload for v2 onwards. class EnvelopeV1(Envelope): """Version 1 of the logging envelope.""" BRAND_ID_FIELD = 'brand_id' DATACENTER_FIELD = 'datacenter' ENVIRONMENT_FIELD = 'environment' ENVELOPE_VERSION_FIELD = Envelope.ENVELOPE_VERSION_FIELD HOST_FIELD = 'host' PAYLOAD_FIELD = 'data' REQUEST_ID_FIELD = 'request_id' ROLE_FIELD = EnvelopeRole.ROLE_FIELD SESSION_ID_FIELD = 'session_id' SEVERITY_FIELD = 'severity' SUB_TYPE_FIELD = 'sub_type' TIMESTAMP_FIELD = 'timestamp' TYPE_FIELD = 'type' DEFAULT_ROLE = EnvelopeRole.ROLE_DEVELOPMENT FIELDS = ( BRAND_ID_FIELD, PAYLOAD_FIELD, DATACENTER_FIELD, ENVELOPE_VERSION_FIELD, ENVIRONMENT_FIELD, HOST_FIELD, REQUEST_ID_FIELD, ROLE_FIELD, SESSION_ID_FIELD, SEVERITY_FIELD, SUB_TYPE_FIELD, TIMESTAMP_FIELD, TYPE_FIELD ) EXTRA_MODIFIABLE_FIELDS = ( PAYLOAD_FIELD, SESSION_ID_FIELD, BRAND_ID_FIELD, SUB_TYPE_FIELD, REQUEST_ID_FIELD, ROLE_FIELD ) EXTRA_IGNORE_FIELDS = (Envelope.ENVELOPE_VERSION_FIELD, Envelope.DATA_SCHEMA_FIELD) def __init__(self, record: logging.LogRecord, envelopeVersion: EnvelopeVersion, configLogData: ConfigLogData, logextra: LogExtra) -> None: super().__init__() self.envelopeVersion = envelopeVersion self.configLogData = configLogData self.set(self.ENVELOPE_VERSION_FIELD, str(envelopeVersion)) self.logextra = logextra self.payloadLogData = None # type: Optional[PayloadLogData] self.load(record) def load(self, record: logging.LogRecord) -> None: super().load(record) self.loadDefaults() self.loadConfig() self.loadRecordFields() self.loadHost() self.loadPayload() self.loadExtra() def loadDefaults(self) -> None: """Load default fields.""" self.set(self.BRAND_ID_FIELD, None) # Optional self.set(self.REQUEST_ID_FIELD, None) # Optional self.set(self.ROLE_FIELD, str(EnvelopeRole())) # Default self.set(self.SESSION_ID_FIELD, None) # Optional def loadConfig(self) -> None: """Load configuration data.""" self.set(self.DATACENTER_FIELD, self.configLogData.get(ConfigLogData.DATACENTER_FIELD)) self.set(self.ENVIRONMENT_FIELD, self.configLogData.get(ConfigLogData.ENVIRONMENT_FIELD)) self.set(self.SUB_TYPE_FIELD, self.configLogData.get(ConfigLogData.SUB_TYPE_FIELD)) # Optional self.set(self.TYPE_FIELD, self.configLogData.get(ConfigLogData.TYPE_FIELD)) def loadPayload(self) -> None: """Load the payload from the log record.""" self.payloadLogData = PayloadLogData(self.record) self.set(self.PAYLOAD_FIELD, self.payloadLogData) def loadHost(self) -> None: """Load the host data.""" self.set(self.HOST_FIELD, HostLogData(self.record)) def loadRecordFields(self) -> None: """Load relevant fields from the log record.""" timestamp = iso8601_utc(datetime.fromtimestamp(self.record.created, pytz.utc)) # type: ignore self.set(self.TIMESTAMP_FIELD, timestamp) self.set(self.SEVERITY_FIELD, self.record.levelname) # type: ignore def loadExtra(self) -> None: """Load any "extra" data.""" if isinstance(self.logextra, LogExtra): try: for k, v in self.logextra.get()[LogExtra.EXTRA_FIELD].items(): if k in self.EXTRA_IGNORE_FIELDS: continue if k in self.EXTRA_MODIFIABLE_FIELDS: if k == self.PAYLOAD_FIELD: if isinstance(v, dict): for key, val in v.items(): self.payloadLogData.set(key, val) # type: ignore else: raise ValueError("Payload is not of type 'dict'") else: self.set(k, v) else: # Place in 'data' block self.payloadLogData.set(k, v) # type: ignore except Exception as e: # pylint: disable=broad-except LOG.error("Unable to process extra log fields: %s", str(e))