CimComplianceReport

CIMReportGenerator

Calculates the statistics of test cases and Creates a MarkDown Report

CIMReportGenerator

Bases: object

Generate the Report. data format::

[
        {
            "data_model": "A",
            "field": "aaa",
            "data_set": "AAA",
            "tag_stanza": "p",
            "status": "pass"/"fail"
        }
]

Parameters:

Name	Type	Description	Default
data	list	List of dictionaries with specified format.	[]

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

class CIMReportGenerator(object):
    """
    Generate the Report.
    data format::

        [
                {
                    "data_model": "A",
                    "field": "aaa",
                    "data_set": "AAA",
                    "tag_stanza": "p",
                    "status": "pass"/"fail"
                }
        ]

    Args:
        data (list): List of dictionaries with specified format.
    """

    def __init__(self, data=[], report_class=MarkDownReport):
        self.data = data
        self.report_generator = report_class()

    def add_data(self, data):
        """
        adds data to object property.

        Args:
            data (list): List of dictionaries with specified format.
        """
        self.data.append(data)

    def _group_by(self, keys, data=None):
        """
        Function to generate group of data using Keys provided

        Args:
            keys (list): Contains keys to group data by.
            data (list): list of dictionaries with specified format.

        Yields:
            data_set.DataSet: data set object mapped with the tags
        """
        if not data:
            data = self.data
        return groupby(
            sorted(data, key=lambda data: data["data_model"]),
            lambda testcase: [testcase[key] for key in keys],
        )

    def _get_count_by(self, keys, data=None):
        """
        Function to generate count of data using Keys provided

        Args:
            keys (list): Contains keys to generate count by.
            data (list): list of dictionaries with specified format.

        Yields:
            data_set.DataSet: data set object mapped with the tags
        """
        for grouped_by, grouped_stats in self._group_by(keys, data):
            yield (
                grouped_by,
                Counter(each["status"] for each in grouped_stats),
            )

    @staticmethod
    def pass_count(counter):
        """
        Function to Get count in Pass/Total format.

        Args:
            counter (collections.Counter): Contains counts of passing/failing Testcases.

        Yields:
            String: string with pass/total format.
        """
        return "{}/{}".format(
            counter["passed"],
            (counter["failed"] + counter["passed"] + counter["skipped"]),
        )

    @staticmethod
    def fail_count(counter):
        """
        Function to Get count in Fail/Total format.

        Args:
            counter (collections.Counter): Contains counts of passing/failing Testcases.

        Yields:
            String: string with fail/total format.
        """
        return "{}/{}".format(
            counter["failed"],
            (counter["failed"] + counter["passed"] + counter["skipped"]),
        )

    def generate_summary_table(self):
        """
        Displays test case summary of the add-on for all the supported data models.
        """
        self.report_generator.add_section_title(" Summary")
        self.report_generator.add_section_description(
            "Displays test case summary of the add-on for all the supported data models."
        )
        summary_table = MarkdownTable("", ["Data Model", "Status", "Fail/Total"])

        data_models = iter(SUPPORTED_DATAMODELS)

        for data_model, stats in self._get_count_by(["data_model"]):
            for each_model in data_models:
                if each_model == data_model[0]:
                    status = "Passed" if stats["failed"] == 0 else "Failed"
                    summary_table.add_row(
                        [data_model[0], status, self.fail_count(stats)]
                    )
                    break
                else:
                    summary_table.add_row([each_model, "N/A", "-"])

        for each in data_models:
            summary_table.add_row([each, "N/A", "-"])

        self.report_generator.add_table(summary_table.return_table_str())

    def generate_tag_stanza_mapping_table(self):
        """
        Displays test case summary for the stanzas in tags.conf and the dataset mapped with it.
        """
        self.report_generator.add_section_title("Tag Stanza Mapping")
        self.report_generator.add_section_description(
            "Displays test case summary for the stanzas in tags.conf and the data model mapped with it."
        )
        tag_stanza_map = MarkdownTable(
            "", ["Tag Stanza", "Data Model", "Data Set", "Fail/Total"]
        )
        for group, stats in self._get_count_by(
            ["tag_stanza", "data_model", "data_set"]
        ):
            tag_stanza, data_model, data_set = group
            tag_stanza_map.add_row(
                [tag_stanza, data_model, data_set, self.fail_count(stats)]
            )

        self.report_generator.add_table(tag_stanza_map.return_table_str())

    def generate_field_summary_table(self):
        """
        Displays test case summary for all the fields in the dataset for the tag-stanza it is mapped with.
        """
        self.report_generator.add_section_title("Field Summary")
        self.report_generator.add_section_description(
            "Displays test case summary for all the fields in the dataset for the tag-stanza it is mapped with."
        )

        for group_name, grouped_data in self._group_by(["tag_stanza", "data_set"]):
            field_summary_table = MarkdownTable(
                " - ".join(group_name),
                ["Field", "Type", "Test Status", "Failure Message"],
            )

            for each_data in grouped_data:
                fields = False
                if each_data["fields"] and not "," in each_data["fields"]:
                    fields = True
                    field_summary_table.add_row(
                        [
                            each_data["fields"],
                            each_data["fields_type"],
                            each_data["status"].title(),
                            each_data["test_property"],
                        ]
                    )
            if not fields:
                field_summary_table.add_row(["-", "-", "-", "-"])
            self.report_generator.add_table(field_summary_table.return_table_str())
            del field_summary_table

    def generate_skip_tests_table(self):
        """
        Displays summary of the skipped tests
        """
        skipped_tests = list(filter(lambda d: d["status"] == "skipped", self.data))
        if skipped_tests:
            skipped_tests_table = MarkdownTable(
                "",
                ["Tag Stanza", "Data Set", "Field"],
            )
            self.report_generator.add_section_title("Skipped Tests Summary")
            for group, stats in self._get_count_by(
                ["tag_stanza", "data_set", "fields"],
                skipped_tests,
            ):
                tag_stanza, data_set, field = group
                if not field:
                    field = "-"
                skipped_tests_table.add_row([tag_stanza, data_set, field])

            self.report_generator.add_table(skipped_tests_table.return_table_str())

    def generate_report(self, report_path):
        """
        Function to generate report from the stored data.

        Args:
            report_path (string): Path to create the report.
        """
        self.report_generator.set_title("CIM AUDIT REPORT")
        self.data.sort(
            key=lambda tc: (
                tc["tag_stanza"],
                tc["data_model"],
                tc["data_set"],
                tc["fields"],
                tc["fields_type"],
            )
        )

        # Generating Summary table.
        self.generate_summary_table()

        # Generating Tag Stanza Mapping table.
        self.generate_tag_stanza_mapping_table()

        # Generating Field Summary tables.
        self.generate_field_summary_table()

        # Generating Skipped tests Table
        self.generate_skip_tests_table()

        # Table for not supported datamodels
        nsd_table = MarkdownTable("Not Supported Datamodels", ["Name"])
        for each_model in NOT_SUPPORTED_DATAMODELS:
            nsd_table.add_row([each_model])
        self.report_generator.add_table(nsd_table.return_table_str())

        # Writing into markdown file
        if self.data:
            self.report_generator.write(report_path)

add_data(data)

adds data to object property.

Parameters:

Name	Type	Description	Default
data	list	List of dictionaries with specified format.	required

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

def add_data(self, data):
    """
    adds data to object property.

    Args:
        data (list): List of dictionaries with specified format.
    """
    self.data.append(data)

fail_count(counter) staticmethod

Function to Get count in Fail/Total format.

Parameters:

Name	Type	Description	Default
counter	Counter	Contains counts of passing/failing Testcases.	required

Yields:

Name	Type	Description
String		string with fail/total format.

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

@staticmethod
def fail_count(counter):
    """
    Function to Get count in Fail/Total format.

    Args:
        counter (collections.Counter): Contains counts of passing/failing Testcases.

    Yields:
        String: string with fail/total format.
    """
    return "{}/{}".format(
        counter["failed"],
        (counter["failed"] + counter["passed"] + counter["skipped"]),
    )

generate_field_summary_table()

Displays test case summary for all the fields in the dataset for the tag-stanza it is mapped with.

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

def generate_field_summary_table(self):
    """
    Displays test case summary for all the fields in the dataset for the tag-stanza it is mapped with.
    """
    self.report_generator.add_section_title("Field Summary")
    self.report_generator.add_section_description(
        "Displays test case summary for all the fields in the dataset for the tag-stanza it is mapped with."
    )

    for group_name, grouped_data in self._group_by(["tag_stanza", "data_set"]):
        field_summary_table = MarkdownTable(
            " - ".join(group_name),
            ["Field", "Type", "Test Status", "Failure Message"],
        )

        for each_data in grouped_data:
            fields = False
            if each_data["fields"] and not "," in each_data["fields"]:
                fields = True
                field_summary_table.add_row(
                    [
                        each_data["fields"],
                        each_data["fields_type"],
                        each_data["status"].title(),
                        each_data["test_property"],
                    ]
                )
        if not fields:
            field_summary_table.add_row(["-", "-", "-", "-"])
        self.report_generator.add_table(field_summary_table.return_table_str())
        del field_summary_table

generate_report(report_path)

Function to generate report from the stored data.

Parameters:

Name	Type	Description	Default
report_path	string	Path to create the report.	required

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

def generate_report(self, report_path):
    """
    Function to generate report from the stored data.

    Args:
        report_path (string): Path to create the report.
    """
    self.report_generator.set_title("CIM AUDIT REPORT")
    self.data.sort(
        key=lambda tc: (
            tc["tag_stanza"],
            tc["data_model"],
            tc["data_set"],
            tc["fields"],
            tc["fields_type"],
        )
    )

    # Generating Summary table.
    self.generate_summary_table()

    # Generating Tag Stanza Mapping table.
    self.generate_tag_stanza_mapping_table()

    # Generating Field Summary tables.
    self.generate_field_summary_table()

    # Generating Skipped tests Table
    self.generate_skip_tests_table()

    # Table for not supported datamodels
    nsd_table = MarkdownTable("Not Supported Datamodels", ["Name"])
    for each_model in NOT_SUPPORTED_DATAMODELS:
        nsd_table.add_row([each_model])
    self.report_generator.add_table(nsd_table.return_table_str())

    # Writing into markdown file
    if self.data:
        self.report_generator.write(report_path)

generate_skip_tests_table()

Displays summary of the skipped tests

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

def generate_skip_tests_table(self):
    """
    Displays summary of the skipped tests
    """
    skipped_tests = list(filter(lambda d: d["status"] == "skipped", self.data))
    if skipped_tests:
        skipped_tests_table = MarkdownTable(
            "",
            ["Tag Stanza", "Data Set", "Field"],
        )
        self.report_generator.add_section_title("Skipped Tests Summary")
        for group, stats in self._get_count_by(
            ["tag_stanza", "data_set", "fields"],
            skipped_tests,
        ):
            tag_stanza, data_set, field = group
            if not field:
                field = "-"
            skipped_tests_table.add_row([tag_stanza, data_set, field])

        self.report_generator.add_table(skipped_tests_table.return_table_str())

generate_summary_table()

Displays test case summary of the add-on for all the supported data models.

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

def generate_summary_table(self):
    """
    Displays test case summary of the add-on for all the supported data models.
    """
    self.report_generator.add_section_title(" Summary")
    self.report_generator.add_section_description(
        "Displays test case summary of the add-on for all the supported data models."
    )
    summary_table = MarkdownTable("", ["Data Model", "Status", "Fail/Total"])

    data_models = iter(SUPPORTED_DATAMODELS)

    for data_model, stats in self._get_count_by(["data_model"]):
        for each_model in data_models:
            if each_model == data_model[0]:
                status = "Passed" if stats["failed"] == 0 else "Failed"
                summary_table.add_row(
                    [data_model[0], status, self.fail_count(stats)]
                )
                break
            else:
                summary_table.add_row([each_model, "N/A", "-"])

    for each in data_models:
        summary_table.add_row([each, "N/A", "-"])

    self.report_generator.add_table(summary_table.return_table_str())

generate_tag_stanza_mapping_table()

Displays test case summary for the stanzas in tags.conf and the dataset mapped with it.

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

def generate_tag_stanza_mapping_table(self):
    """
    Displays test case summary for the stanzas in tags.conf and the dataset mapped with it.
    """
    self.report_generator.add_section_title("Tag Stanza Mapping")
    self.report_generator.add_section_description(
        "Displays test case summary for the stanzas in tags.conf and the data model mapped with it."
    )
    tag_stanza_map = MarkdownTable(
        "", ["Tag Stanza", "Data Model", "Data Set", "Fail/Total"]
    )
    for group, stats in self._get_count_by(
        ["tag_stanza", "data_model", "data_set"]
    ):
        tag_stanza, data_model, data_set = group
        tag_stanza_map.add_row(
            [tag_stanza, data_model, data_set, self.fail_count(stats)]
        )

    self.report_generator.add_table(tag_stanza_map.return_table_str())

pass_count(counter) staticmethod

Function to Get count in Pass/Total format.

Parameters:

Name	Type	Description	Default
counter	Counter	Contains counts of passing/failing Testcases.	required

Yields:

Name	Type	Description
String		string with pass/total format.

Source code in pytest_splunk_addon/cim_compliance/cim_report_generator.py

@staticmethod
def pass_count(counter):
    """
    Function to Get count in Pass/Total format.

    Args:
        counter (collections.Counter): Contains counts of passing/failing Testcases.

    Yields:
        String: string with pass/total format.
    """
    return "{}/{}".format(
        counter["passed"],
        (counter["failed"] + counter["passed"] + counter["skipped"]),
    )

MarkDownReport

Markdown generator

MarkDownReport

Bases: CIMReport

Generate the markdown content

Source code in pytest_splunk_addon/cim_compliance/markdown_report.py

class MarkDownReport(CIMReport):
    """
    Generate the markdown content
    """

    def __init__(self):
        self.markdown_str = ""
        self.note_str = ""

    def set_title(self, title_string):
        """
        Function to set title of a report

        Args:
            title_string (str): String containing title for report.
        """
        self.title_str = "# {} \n".format(title_string)

    def add_section_title(self, section_title):
        """
        Function to add new section to report

        Args:
            section_title (str): String containing title for new Section.
        """
        self.markdown_str += "\n## {}\n".format(section_title)

    def add_section_description(self, description):
        """
        Adds description string to the section

        Args:
            description (str): Description string.
        """
        self.markdown_str += "\n**Description:** " + description + "\n"

    def add_section_note(self, section_note):
        """
        Function to set Note in a report

        Args:
            section_note (str): String containing note for report.
        """
        self.note_str = "## Note: {} \n".format(section_note)

    def add_table(self, table_string):
        """
        Function to add a table to the Report.

        Args:
            table_string (str): Stringified table.
        """
        self.markdown_str += table_string

    def write(self, path):
        """
        Function to add a table to the Report.

        Args:
            path (str) : path to store report file.
        """
        with open(path, "w") as report:
            report.write(self.title_str)
            report.write(self.markdown_str)
            report.write(self.note_str)

add_section_description(description)

Adds description string to the section

Parameters:

Name	Type	Description	Default
description	str	Description string.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_report.py

def add_section_description(self, description):
    """
    Adds description string to the section

    Args:
        description (str): Description string.
    """
    self.markdown_str += "\n**Description:** " + description + "\n"

add_section_note(section_note)

Function to set Note in a report

Parameters:

Name	Type	Description	Default
section_note	str	String containing note for report.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_report.py

def add_section_note(self, section_note):
    """
    Function to set Note in a report

    Args:
        section_note (str): String containing note for report.
    """
    self.note_str = "## Note: {} \n".format(section_note)

add_section_title(section_title)

Function to add new section to report

Parameters:

Name	Type	Description	Default
section_title	str	String containing title for new Section.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_report.py

def add_section_title(self, section_title):
    """
    Function to add new section to report

    Args:
        section_title (str): String containing title for new Section.
    """
    self.markdown_str += "\n## {}\n".format(section_title)

add_table(table_string)

Function to add a table to the Report.

Parameters:

Name	Type	Description	Default
table_string	str	Stringified table.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_report.py

def add_table(self, table_string):
    """
    Function to add a table to the Report.

    Args:
        table_string (str): Stringified table.
    """
    self.markdown_str += table_string

set_title(title_string)

Function to set title of a report

Parameters:

Name	Type	Description	Default
title_string	str	String containing title for report.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_report.py

def set_title(self, title_string):
    """
    Function to set title of a report

    Args:
        title_string (str): String containing title for report.
    """
    self.title_str = "# {} \n".format(title_string)

write(path)

Function to add a table to the Report.

Parameters:

Name	Type	Description	Default
path	str)	path to store report file.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_report.py

def write(self, path):
    """
    Function to add a table to the Report.

    Args:
        path (str) : path to store report file.
    """
    with open(path, "w") as report:
        report.write(self.title_str)
        report.write(self.markdown_str)
        report.write(self.note_str)

MarkDownTable

Markdown table generator

MarkdownTable

Bases: BaseTable

Generate table in markdown format

Parameters:

Name	Type	Description	Default
table_title	str	Title of the table	required
header_list	list(str	List of header names	required

Source code in pytest_splunk_addon/cim_compliance/markdown_table.py

class MarkdownTable(BaseTable):
    """
    Generate table in markdown format

    Args:
        table_title (str): Title of the table
        header_list (list(str)): List of header names
    """

    def __init__(self, table_title, header_list):
        self.table_title = self.__set_title(table_title)
        self.table_headers = self.__set_headers(header_list)
        self.row_str = ""
        self.table_description = ""
        self.table_note = ""

    def __set_title(self, title):
        """
        Adds Title string to the table

        Args:
            title (str): Title string.
        """
        return "### {}".format(title) if title else ""

    def __set_headers(self, header_list):
        """
        Sets the header column for the table.

        Args:
            header_list (list): Contains list of column headers.
        """
        header_str = "\n"
        helper_str = ""
        for each_column in header_list:
            header_str += "| {} ".format(each_column)
            helper_str += "|:{}".format("-" * len(each_column))
        return "{} |\n {} |\n".format(header_str, helper_str)

    def set_description(self, description):
        """
        Adds description string to the table

        Args:
            description (str): Description string.
        """
        self.table_description = "\n {} \n".format(description)

    def add_row(self, value_list):
        """
        Expects a list of row values to be added in the table

        Args:
            value_list (list): Contains list of row values
        """
        row_element = ""
        for each_value in value_list:
            row_element += "| {} ".format(each_value)
        self.row_str += "{} |\n".format(row_element)

    def set_note(self, note_str):
        """
        It adds the note at the end of the table

        Args:
            note_str (str): Note string to be added.
        """
        self.table_note = "\n*NOTE: {} *\n ".format(note_str)

    def return_table_str(self):
        """
        Generates the final table str
        """
        self.table_str = self.table_title
        self.table_str += self.table_description
        self.table_str += self.table_headers
        self.table_str += self.row_str
        self.table_str += self.table_note
        return self.table_str + "\n"

__set_headers(header_list)

Sets the header column for the table.

Parameters:

Name	Type	Description	Default
header_list	list	Contains list of column headers.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_table.py

def __set_headers(self, header_list):
    """
    Sets the header column for the table.

    Args:
        header_list (list): Contains list of column headers.
    """
    header_str = "\n"
    helper_str = ""
    for each_column in header_list:
        header_str += "| {} ".format(each_column)
        helper_str += "|:{}".format("-" * len(each_column))
    return "{} |\n {} |\n".format(header_str, helper_str)

__set_title(title)

Adds Title string to the table

Parameters:

Name	Type	Description	Default
title	str	Title string.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_table.py

def __set_title(self, title):
    """
    Adds Title string to the table

    Args:
        title (str): Title string.
    """
    return "### {}".format(title) if title else ""

add_row(value_list)

Expects a list of row values to be added in the table

Parameters:

Name	Type	Description	Default
value_list	list	Contains list of row values	required

Source code in pytest_splunk_addon/cim_compliance/markdown_table.py

def add_row(self, value_list):
    """
    Expects a list of row values to be added in the table

    Args:
        value_list (list): Contains list of row values
    """
    row_element = ""
    for each_value in value_list:
        row_element += "| {} ".format(each_value)
    self.row_str += "{} |\n".format(row_element)

return_table_str()

Generates the final table str

Source code in pytest_splunk_addon/cim_compliance/markdown_table.py

def return_table_str(self):
    """
    Generates the final table str
    """
    self.table_str = self.table_title
    self.table_str += self.table_description
    self.table_str += self.table_headers
    self.table_str += self.row_str
    self.table_str += self.table_note
    return self.table_str + "\n"

set_description(description)

Adds description string to the table

Parameters:

Name	Type	Description	Default
description	str	Description string.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_table.py

def set_description(self, description):
    """
    Adds description string to the table

    Args:
        description (str): Description string.
    """
    self.table_description = "\n {} \n".format(description)

set_note(note_str)

It adds the note at the end of the table

Parameters:

Name	Type	Description	Default
note_str	str	Note string to be added.	required

Source code in pytest_splunk_addon/cim_compliance/markdown_table.py

def set_note(self, note_str):
    """
    It adds the note at the end of the table

    Args:
        note_str (str): Note string to be added.
    """
    self.table_note = "\n*NOTE: {} *\n ".format(note_str)