Sensitive Data (e.g. PII) Scanner

import csv
import argparse
from kada_collectors.extractors.utils import load_config, get_generic_logger
from kada_collectors.extractors.pii_scanner import PIIScanner, VALID_DEFAULT_DETECTORS

get_generic_logger('root') # Set to use the root logger, you can change the context accordingly or define your own logger

parser = argparse.ArgumentParser(description='KADA PII Scanner.')
parser.add_argument('--extractor-config', '-e', dest='extractor_config', type=str, required=True, help='Location of the extarctor configuration json.')
parser.add_argument('--objects-file-path', '-f', dest='objects_file_path', type=str, required=True, help='Location of the .txt file that contains the list of objects to scan for the source.')
parser.add_argument('--source-type', '-t', dest='source_type', type=str, required=True, help='What kind of source are we scanning? E.g. snowflake, oracle etc. See documentation for full list of supported source types.')
parser.add_argument('--sample-size', '-s', dest='sample_size', type=int, required=True, help='How many rows are we sampling for each object, note that 0 means all rows will be sampled, number should be greater or equal to 0.')
parser.add_argument('--parrallel', '-p', dest='concurrency', type=int, default=1, help='Should it be running in parallel, default is 1 meaning no parallelism, consider your CPU resources when settings this value.')
parser.add_argument('--default-detectors', '-d', dest='default_detectors', type=str, help='Comma seperated list of default detectors the scanner should use {}'.format(','.join(VALID_DEFAULT_DETECTORS)))
parser.add_argument('--delta', '-a', dest='delta', action='store_true', help='Produces a DELTA extract file if you are doing a partial scan.')
parser.add_argument('--pii-output-path', '-o', dest='pii_output_path', type=str, required=True, help='The output path to the folder where the extract should appear.')
args = parser.parse_args()

# ######
# Impliment additional logic for checking existence if you wish
# You may also choose to call the PIIScanner differently and not use an input file this is completely up to you
# You can also feed in from your own custom producer for the list of objects
# This is simply the default out of the box implimentation to call the PIIScanner and produce the required Extract File for K
# ######

def read_validate_object_file(file_path):
    """
    Reads the flat file and validates the header and returns an iterator
    This is simply the out of the box way to feed the scanner, you may choose a different
    way to feed the scanner
    """
    with open (file_path, 'r', encoding='utf-8') as csv_file:
        reader = csv.reader(csv_file, delimiter=',')
        header = next(reader) # Skip the header
        if [x.upper() for x in header] != ['OBJECT_TYPE','OBJECT_ID']: # Should be a flat file thats comma delimited with the headers OBJECT_TYPE and OBJECT_ID
            raise Exception('Invalid object file')
        return [x for x in reader] # Return a list not an iterator as we will close the file

if __name__ == '__main__': # Do not omit this syntax as the Class impliments multiprocessing
    extractor_config = load_config(args.extractor_config) # Load the corresponding collector config file
    object_list = read_validate_object_file(args.objects_file_path) # 2D Array of objects
    # You can define your own Detector classes and register them before calling .scan() method to ensure the scanner picks up the new Detector Class, read the documentation on how to impliment new classes
    # You'll need to decorate the class with kada_collectors.extractors.pii_scanner.register_detector 
    default_detectors = [x.strip() for x in args.default_detectors.split(',')] if args.default_detectors else []
    pii_scanner = PIIScanner(args.source_type, args.sample_size, args.concurrency, object_list, args.pii_output_path, default_detectors=default_detectors, delta=args.delta, **extractor_config)
    pii_scanner.scan()

python kada_pii_scanner.py -e ./kada_snowflake_extractor_config.json -f ./pii_test_scan.csv -t snowflake -s 10 -p 8 -o /tmp/output -d Email,AUPhoneNumber,CreditCard,AUTaxFileNumber,AUZipCode,AUDriversLicense,AUBSBNumber,AUBankAccountNumber,AUMedicare,AUPassport

python kada_pii_scanner.py --help

from kada_collectors.extractors.pii_scanner import PIIScanner

if __name__ == '__main__': # Ensure you have the main syntax or the caller has the main syntax
  pii_scanner = PIIScanner(source_type, sample_size, concurrency, object_list, output_path, disable_defaults=False, **extractor_config)
  for dets in pii_scanner.registered_detectors.values():
    print(dets.keys())

from kada_collectors.extractors.pii_scanner import PIIScanner, register_detector, DatabaseDatumDetectors, Email

@register_detector
class MyEmailDetector(DatabaseDatumDetectors):
    """
    Example implimentation of your own Detector
    """
    
    def detect(self, datum, column_name):
        """
        datum: is the data that needs to checked, its up to the detector implimentation to transform the incoming dbapi value type and skip if required
        column_name: is the name of the column of which the data belongs to
        
        returns: A list of PIIType Instances that are matched
        """
        matches = []
        if not isinstance(datum, bool): # We only don't apply to boolean
            if '@' in str(datum):
                matches.append(pii_cls())
        return matches
        
        
# Call Scanner class as normal
if __name__ == '__main__': # Ensure you have the main syntax or the caller has the main syntax
  pii_scanner = PIIScanner(source_type, sample_size, concurrency, object_list, output_path, disable_defaults=False, **extractor_config)
  pii_scanner.scan()

from mypackageofdetectors import MyDetector
from kada_collectors.extractors.pii_scanner register_detector

register_detector(MyDetector)

import csv
import argparse
from kada_collectors.extractors.utils import load_config, get_generic_logger
from kada_collectors.extractors.pii_scanner import PIIScanner, register_detector, DatabaseDatumDetectors, PIIType

get_generic_logger('root') # Set to use the root logger, you can change the context accordingly or define your own logger

parser = argparse.ArgumentParser(description='KADA PII Scanner.')
parser.add_argument('--extractor-config', '-e', dest='extractor_config', type=str, required=True, help='Location of the extarctor configuration json.')
parser.add_argument('--objects-file-path', '-f', dest='objects_file_path', type=str, required=True, help='Location of the .txt file that contains the list of objects to scan for the source.')
parser.add_argument('--source-type', '-t', dest='source_type', type=str, required=True, help='What kind of source are we scanning? E.g. snowflake, oracle etc. See documentation for full list of supported source types.')
parser.add_argument('--sample-size', '-s', dest='sample_size', type=int, required=True, help='How many rows are we sampling for each object, note that 0 means all rows will be sampled, number should be greater or equal to 0.')
parser.add_argument('--parrallel', '-p', dest='concurrency', type=int, default=1, help='Should it be running in parallel, default is 1 meaning no parallelism, consider your CPU resources when settings this value.')
parser.add_argument('--default-detectors', '-d', dest='default_detectors', type=str, help='Comma seperated list of default detectors the scanner should use {}'.format(','.join(VALID_DEFAULT_DETECTORS)))
parser.add_argument('--delta', '-a', dest='delta', action='store_true', help='Produces a DELTA extract file if you are doing a partial scan.')
parser.add_argument('--pii-output-path', '-o', dest='pii_output_path', type=str, required=True, help='The output path to the folder where the extract should appear.')
args = parser.parse_args()

# ######
# Impliment additional logic for checking existence if you wish
# You may also choose to call the PIIScanner differently and not use an input file this is completely up to you
# You can also feed in from your own custom producer for the list of objects
# This is simply the default out of the box implimentation to call the PIIScanner and produce the required Extract File for K
# ######

def read_validate_object_file(file_path):
    """
    Reads the flat file and validates the header and returns an iterator
    This is simply the out of the box way to feed the scanner, you may choose a different
    way to feed the scanner
    """
    with open (file_path, 'r', encoding='utf-8') as csv_file:
        reader = csv.reader(csv_file, delimiter=',')
        header = next(reader) # Skip the header
        if [x.upper() for x in header] != ['OBJECT_TYPE','OBJECT_ID']: # Should be a flat file thats comma delimited with the headers OBJECT_TYPE and OBJECT_ID
            raise Exception('Invalid object file')
        return [x for x in reader] # Return a list not an iterator as we will close the file

class MyType(PIIType):
    name = 'Me'

@register_detector
class MyEmailDetector(DatabaseDatumDetectors):
    """
    Example implimentation of your own Detector
    """
    
    def detect(self, datum, column_name):
        """
        datum: is the data that needs to checked, its up to the detector implimentation to transform the incoming dbapi value type and skip if required
        
        returns: A list of PIIType Instances that are matched
        """
        matches = []
        if not isinstance(datum, bool): # We only don't apply to boolean
            if '@' in str(datum):
                matches.append(MyType())
        return matches

if __name__ == '__main__': # Do not omit this syntax as the Class impliments multiprocessing
    extractor_config = load_config(args.extractor_config) # Load the corresponding collector config file
    object_list = read_validate_object_file(args.objects_file_path) # 2D Array of objects
    # You can define your own Detector classes and register them before calling .scan() method to ensure the scanner picks up the new Detector Class, read the documentation on how to impliment new classes
    # You'll need to decorate the class with kada_collectors.extractors.pii_scanner.register_detector 
    pii_scanner = PIIScanner(args.source_type, args.sample_size, args.concurrency, object_list, args.pii_output_path, disable_defaults=args.disable_defaults, **extractor_config)
    pii_scanner.scan()

from kada_collectors.extractors.pii_scanner import PIIScanner

if __name__ == '__main__': # Ensure you have the main syntax or the caller has the main syntax
  extractor_config = {# Your own way of handling extractor configs e.g. ENV variables}
  source_type = 'snowflake' # Defintion of these fields are as above in the wrapper table, otherwise see the class defintion below
  sample_size = 1
  concurrency = 1 
  object_list = [] # This is a 2D array of objects in the form [[<object_type>, <object_id>]]
  pii_scanner = PIIScanner(source_type, sample_size, concurrency, object_list, pii_output_path, disable_defaults=False, detal=False, **extractor_config)
  pii_scanner.scan()

PIIScanner(source_type: str,
    rows: int,
    concurrency: int,
    object_list: list,
    pii_output_path: str,
    *args: Any,
    default_detectors: list = [],
    delta: bool = False,
    **kwargs: Any) -> None

pii_scanner = PIIScanner('snowflake', 10, 1, [], **extractor_config)
payloads = pii_scanner.scan_data(object_list)

object_list = [['TABLE','kada_snowflakecomputing_com.dwh.extract.tableA']]

[['COLUMN','kada_snowflakecomputing_com.dwh.extract.tableA.col1','3dbf7c4f-1d05-3b75-9920-9122e6eaed56','my detected pii']]

final_payloads = payloads1 + payloads2

pii_scanner = PIIScanner('snowflake', 10, 1, [], **extractor_config)
pii_scanner.scan_call_back(final_payloads)
pii_scanner.produce_extract()

[['COLUMN','kada_snowflakecomputing_com.dwh.extract.tableA.col1','3dbf7c4f-1d05-3b75-9920-9122e6eaed56','my detected pii']]