"""""" import csv from dataclasses import dataclass from datetime import datetime from enum import Enum import glob import re import logging import os from typing import List, Tuple from registrar.models.transition_domain import TransitionDomain from .epp_data_containers import ( AgencyAdhoc, DomainAdditionalData, DomainEscrow, DomainTypeAdhoc, OrganizationAdhoc, AuthorityAdhoc, EnumFilenames, ) from .transition_domain_arguments import TransitionDomainArguments from .terminal_helper import TerminalColors, TerminalHelper, LogCode logger = logging.getLogger(__name__) class FileTransitionLog: """Container for storing event logs. Used to lessen the complexity of storing multiple logs across multiple variables. self.logs: dict -> { EnumFilenames.DOMAIN_ADHOC: List[LogItem], EnumFilenames.AGENCY_ADHOC: List[LogItem], EnumFilenames.ORGANIZATION_ADHOC: List[LogItem], EnumFilenames.DOMAIN_ADDITIONAL: List[LogItem], } """ def __init__(self): self.logs = {} class LogItem: """Used for storing data about logger information.""" def __init__(self, file_type, code, message, domain_name): self.file_type = file_type self.code = code self.message = message self.domain_name = domain_name def add_log(self, file_type, code, message, domain_name): """Adds a log item to self.logs file_type -> Which enum to associate with, ex. EnumFilenames.DOMAIN_ADHOC code -> Log severity or other metadata, ex. LogCode.ERROR message -> Message to display domain_name -> Name of the domain, i.e. "igorville.gov" """ log = self.LogItem(file_type, code, message, domain_name) dict_name = (file_type, domain_name) self._add_to_log_list(dict_name, log) def create_log_item( self, file_type, code, message, domain_name=None, add_to_list=True, minimal_logging=True ): """Creates and returns an LogItem object. add_to_list: bool -> If enabled, add it to the logs array. """ log = self.LogItem(file_type, code, message, domain_name) if not add_to_list: return log dict_name = (file_type, domain_name) self._add_to_log_list(dict_name, log) restrict_type = [] if minimal_logging: restrict_type = [LogCode.INFO, LogCode.WARNING] TerminalHelper.print_conditional( log.code not in restrict_type, log.message, log.code, ) return log def _add_to_log_list(self, log_name, log): if log_name not in self.logs: self.logs[log_name] = [log] else: self.logs[log_name].append(log) def display_all_logs(self): """Logs every LogItem contained in this object""" for parent_log in self.logs: for child_log in parent_log: TerminalHelper.print_conditional( True, child_log.message, child_log.severity ) def display_logs_by_domain_name(self, domain_name, restrict_type=LogCode.DEFAULT): """Displays all logs of a given domain_name. Will log with the correct severity depending on code. domain_name: str -> The domain to target, such as "igorville.gov" restrict_type: LogCode -> Determines if only errors of a certain type should be displayed, such as LogCode.ERROR. """ for file_type in EnumFilenames: domain_logs = self.get_logs(file_type, domain_name) if domain_logs is None: return None for log in domain_logs: TerminalHelper.print_conditional( restrict_type != log.code, log.message, log.code ) def get_logs(self, file_type, domain_name): """Grabs the logs associated with a particular file_type and domain_name""" log_name = (file_type, domain_name) return self.logs.get(log_name) class LoadExtraTransitionDomain: """Grabs additional data for TransitionDomains.""" def __init__(self, options: TransitionDomainArguments): # Globally stores event logs and organizes them self.parse_logs = FileTransitionLog() self.debug = options.debug # Reads and parses migration files self.parsed_data_container = ExtraTransitionDomain(options) self.parsed_data_container.parse_all_files(options.infer_filenames) def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content given in self.parsed_data_container""" all_transition_domains = TransitionDomain.objects.all() if not all_transition_domains.exists(): raise ValueError("No TransitionDomain objects exist.") updated_transition_domains = [] failed_transition_domains = [] for transition_domain in all_transition_domains: domain_name = transition_domain.domain_name updated_transition_domain = transition_domain try: # STEP 1: Parse organization data updated_transition_domain = self.parse_org_data( domain_name, transition_domain ) # STEP 2: Parse domain type data updated_transition_domain = self.parse_domain_type_data( domain_name, transition_domain ) # STEP 3: Parse agency data updated_transition_domain = self.parse_agency_data( domain_name, transition_domain ) # STEP 4: Parse creation and expiration data updated_transition_domain = self.parse_creation_expiration_data( domain_name, transition_domain ) # Check if the instance has changed before saving #if updated_transition_domain.__dict__ != transition_domain.__dict__: updated_transition_domain.save() updated_transition_domains.append(updated_transition_domain) logger.info( f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" f"{TerminalColors.ENDC}" ) # If we run into an exception on this domain, # Just skip over it and log that it happened. # Q: Should we just throw an exception? except Exception as err: logger.debug(err) logger.error( f"{TerminalColors.FAIL}" f"Exception encountered on {domain_name}. Could not update." f"{TerminalColors.ENDC}" ) failed_transition_domains.append(domain_name) if self.debug: # Display misc errors (not associated to a domain) self.parse_logs.display_logs_by_domain_name(None) failed_count = len(failed_transition_domains) if failed_count == 0: TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}") logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== Updated {len(updated_transition_domains)} transition domain entries: {TerminalColors.ENDC} """ ) else: # TODO - update TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}") logger.error( f"""{TerminalColors.FAIL} ============= FINISHED WITH ERRORS =============== Updated {len(updated_transition_domains)} transition domain entries, Failed to update {failed_count} transition domain entries {TerminalColors.ENDC} """ ) # TODO if TransitionDomain.objects.all().count() != len(updated_transition_domains): logger.error("Something bad happened") def parse_creation_expiration_data(self, domain_name, transition_domain): """Grabs expiration_date from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") info = self.get_domain_escrow_info(domain_name) if info is None: self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ESCROW, LogCode.ERROR, "Could not add epp_creation_date and epp_expiration_date " f"on {domain_name}, no data exists.", domain_name, not self.debug ) return transition_domain creation_exists = ( transition_domain.epp_creation_date is not None ) expiration_exists = ( transition_domain.epp_expiration_date is not None ) transition_domain.epp_creation_date = info.creationdate transition_domain.epp_expiration_date = info.expirationdate # Logs if we either added to this property, # or modified it. self._add_or_change_message( EnumFilenames.DOMAIN_ESCROW, "epp_creation_date", transition_domain.epp_creation_date, domain_name, creation_exists, ) self._add_or_change_message( EnumFilenames.DOMAIN_ESCROW, "epp_expiration_date", transition_domain.epp_expiration_date, domain_name, expiration_exists, ) return transition_domain def parse_agency_data(self, domain_name, transition_domain) -> TransitionDomain: """Grabs federal_agency from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") info = self.get_agency_info(domain_name) if info is None: self.parse_logs.create_log_item( EnumFilenames.AGENCY_ADHOC, LogCode.ERROR, f"Could not add federal_agency on {domain_name}, no data exists.", domain_name, not self.debug ) return transition_domain agency_exists = ( transition_domain.federal_agency is not None and transition_domain.federal_agency.strip() != "" ) if not info.active.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add inactive agency {info.agencyname} on {domain_name}", domain_name, not self.debug ) return transition_domain if not info.isfederal.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add non-federal agency {info.agencyname} on {domain_name}", domain_name, not self.debug ) return transition_domain transition_domain.federal_agency = info.agencyname # Logs if we either added to this property, # or modified it. self._add_or_change_message( EnumFilenames.AGENCY_ADHOC, "federal_agency", transition_domain.federal_agency, domain_name, agency_exists, ) return transition_domain def parse_domain_type_data( self, domain_name, transition_domain: TransitionDomain ) -> TransitionDomain: """Grabs organization_type and federal_type from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") info = self.get_domain_type_info(domain_name) if info is None: self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add domain_type on {domain_name}, no data exists.", domain_name, not self.debug ) return transition_domain # This data is stored as follows: FEDERAL - Judicial # For all other records, it is stored as so: Interstate # We can infer if it is federal or not based on this fact. domain_type = info.domaintype.split("-") domain_type_length = len(domain_type) if domain_type_length < 1 or domain_type_length > 2: raise ValueError("Found invalid data on DOMAIN_ADHOC") # Then, just grab the organization type. new_organization_type = domain_type[0].strip() # Check if this domain_type is active or not. # If not, we don't want to add this. if not info.active.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", domain_name, not self.debug ) return transition_domain # Are we updating data that already exists, # or are we adding new data in its place? organization_type_exists = ( transition_domain.organization_type is not None and transition_domain.organization_type.strip() != "" ) federal_type_exists = ( transition_domain.federal_type is not None and transition_domain.federal_type.strip() != "" ) # If we get two records, then we know it is federal. # needs to be lowercase for federal type is_federal = domain_type_length == 2 if is_federal: new_federal_type = domain_type[1].strip() transition_domain.organization_type = new_organization_type transition_domain.federal_type = new_federal_type else: transition_domain.organization_type = new_organization_type transition_domain.federal_type = None # Logs if we either added to this property, # or modified it. self._add_or_change_message( EnumFilenames.DOMAIN_ADHOC, "organization_type", transition_domain.organization_type, domain_name, organization_type_exists, ) self._add_or_change_message( EnumFilenames.DOMAIN_ADHOC, "federal_type", transition_domain.federal_type, domain_name, federal_type_exists, ) return transition_domain def parse_org_data( self, domain_name, transition_domain: TransitionDomain ) -> TransitionDomain: """Grabs organization_name from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") org_info = self.get_org_info(domain_name) if org_info is None: self.parse_logs.create_log_item( EnumFilenames.ORGANIZATION_ADHOC, LogCode.ERROR, f"Could not add organization_name on {domain_name}, no data exists.", domain_name, not self.debug ) return transition_domain desired_property_exists = ( transition_domain.organization_name is not None and transition_domain.organization_name.strip() != "" ) transition_domain.organization_name = org_info.orgname # Logs if we either added to this property, # or modified it. self._add_or_change_message( EnumFilenames.ORGANIZATION_ADHOC, "organization_name", transition_domain.organization_name, domain_name, desired_property_exists, ) return transition_domain def _add_or_change_message( self, file_type, var_name, changed_value, domain_name, is_update=False ): """Creates a log instance when a property is successfully changed on a given TransitionDomain.""" if not is_update: self.parse_logs.create_log_item( file_type, LogCode.INFO, f"Added {var_name} as '{changed_value}' on {domain_name}", domain_name, not self.debug ) else: self.parse_logs.create_log_item( file_type, LogCode.WARNING, f"Updated existing {var_name} to '{changed_value}' on {domain_name}", domain_name, not self.debug ) # Property getters, i.e. orgid or domaintypeid def get_org_info(self, domain_name) -> OrganizationAdhoc: """Maps an id given in get_domain_data to a organization_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: return None org_id = domain_info.orgid return self.get_organization_adhoc(org_id) def get_domain_type_info(self, domain_name) -> DomainTypeAdhoc: """Maps an id given in get_domain_data to a domain_type_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: return None type_id = domain_info.domaintypeid return self.get_domain_adhoc(type_id) def get_agency_info(self, domain_name) -> AgencyAdhoc: """Maps an id given in get_domain_data to a agency_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: return None # The agency record is within the authority adhoc authority_id = domain_info.authorityid authority = self.get_authority_adhoc(authority_id) type_id = None if authority is not None: type_id = authority.agencyid return self.get_agency_adhoc(type_id) def get_authority_info(self, domain_name): """Maps an id given in get_domain_data to a authority_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: return None type_id = domain_info.authorityid return self.get_authority_adhoc(type_id) def get_domain_escrow_info(self, domain_name): domain_info = self.get_domain_data(domain_name) if domain_info is None: return None type_id = domain_info.domainname return self.get_domain_escrow(type_id) # Object getters, i.e. DomainAdditionalData or OrganizationAdhoc def get_domain_data(self, desired_id) -> DomainAdditionalData: """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: """Grabs a corresponding row within the ORGANIZATION_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) def get_domain_adhoc(self, desired_id) -> DomainTypeAdhoc: """Grabs a corresponding row within the DOMAIN_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.DOMAIN_ADHOC, desired_id) def get_agency_adhoc(self, desired_id) -> AgencyAdhoc: """Grabs a corresponding row within the AGENCY_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.AGENCY_ADHOC, desired_id) def get_authority_adhoc(self, desired_id) -> AuthorityAdhoc: """Grabs a corresponding row within the AUTHORITY_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.AUTHORITY_ADHOC, desired_id) def get_domain_escrow(self, desired_id) -> DomainEscrow: """Grabs a corresponding row within the DOMAIN_ESCROW file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.DOMAIN_ESCROW, desired_id) # TODO - renamed / needs a return section def get_object_by_id(self, file_type: EnumFilenames, desired_id): """Returns a field in a dictionary based off the type and id. vars: file_type: (constant) EnumFilenames -> Which data file to target. An example would be `EnumFilenames.DOMAIN_ADHOC`. desired_id: str -> Which id you want to search on. An example would be `"12"` or `"igorville.gov"` Explanation: Each data file has an associated type (file_type) for tracking purposes. Each file_type is a dictionary which contains a dictionary of row[id_field]: object. In practice, this would look like: EnumFilenames.AUTHORITY_ADHOC: { "1": AuthorityAdhoc(...), "2": AuthorityAdhoc(...), ... } desired_id will then specify which id to grab. If we wanted "1", then this function will return the value of id "1". So, `AuthorityAdhoc(...)` """ # Grabs a dict associated with the file_type. # For example, EnumFilenames.DOMAIN_ADDITIONAL. desired_type = self.parsed_data_container.file_data.get(file_type) if desired_type is None: self.parse_logs.create_log_item( file_type, LogCode.ERROR, f"Type {file_type} does not exist", ) return None # Grab the value given an Id within that file_type dict. # For example, "igorville.gov". obj = desired_type.data.get(desired_id) if obj is None: self.parse_logs.create_log_item( file_type, LogCode.ERROR, f"Id {desired_id} does not exist" ) return obj # TODO - change name @dataclass class PatternMap: """Helper class that holds data and metadata about a requested file. filename: str -> The desired filename to target. If no filename is given, it is assumed that you are passing in a filename pattern and it will look for a filename that matches the given postfix you pass in. regex: re.Pattern -> Defines what regex you want to use when inferring filenames. If none, no matching occurs. data_type: type -> Metadata about the desired type for data. id_field: str -> Defines which field should act as the id in data. This is necessary as we store lists of "data_type" in ExtraTransitionDomain as follows: { id_field: data_type(...), id_field: data_type(...), ... } """ def __init__( self, filename: str, regex: re.Pattern, data_type: type, id_field: str, ): # Metadata # ## Filename inference metadata ## self.regex = regex self.could_infer = False ## "data" object metadata ## ### Where the data is sourced from ### self.filename = filename ### What type the data is ### self.data_type = data_type ### What the id should be in the holding dict ### # TODO - rename to id_field_name self.id_field = id_field # Object data # self.data = {} def try_infer_filename(self, current_file_name, default_file_name): """Tries to match a given filename to a regex, then uses that match to generate the filename.""" # returns (filename, inferred_successfully) return self._infer_filename(self.regex, current_file_name, default_file_name) def _infer_filename(self, regex: re.Pattern, matched_file_name, default_file_name): if not isinstance(regex, re.Pattern): return (self.filename, False) match = regex.match(matched_file_name) if not match: return (self.filename, False) total_groups = len(match.groups()) # If no matches exist or if we have too many # matches, then we shouldn't infer if total_groups == 0 or total_groups > 2: return (self.filename, False) # If only one match is returned, # it means that our default matches our request if total_groups == 1: return (self.filename, True) # Otherwise, if two are returned, then # its likely the pattern we want date = match.group(1) filename_without_date = match.group(2) # After stripping out the date, # do the two filenames match? can_infer = filename_without_date == default_file_name if not can_infer: return (self.filename, False) # If they do, recreate the filename and return it full_filename = date + "." + filename_without_date return (full_filename, can_infer) class ExtraTransitionDomain: """Helper class to aid in storing TransitionDomain data spread across multiple files.""" strip_date_regex = re.compile(r"(?:.*\/)?(\d+)\.(.+)") def __init__(self, options: TransitionDomainArguments): # Add a slash if the last character isn't one if options.directory and options.directory[-1] != "/": options.directory += "/" self.directory = options.directory self.seperator = options.sep self.all_files = glob.glob(f"{self.directory}*") # Create a set with filenames as keys for quick lookup self.all_files_set = {os.path.basename(file) for file in self.all_files} # Used for a container of values at each filename. # Instead of tracking each in a seperate variable, we can declare # metadata about each file and associate it with an enum. # That way if we want the data located at the agency_adhoc file, # we can just call EnumFilenames.AGENCY_ADHOC. pattern_map_params = [ ( EnumFilenames.AGENCY_ADHOC, options.agency_adhoc_filename, AgencyAdhoc, "agencyid", ), ( EnumFilenames.DOMAIN_ADDITIONAL, options.domain_additional_filename, DomainAdditionalData, "domainname", ), ( EnumFilenames.DOMAIN_ESCROW, options.domain_escrow_filename, DomainEscrow, "domainname", ), ( EnumFilenames.DOMAIN_ADHOC, options.domain_adhoc_filename, DomainTypeAdhoc, "domaintypeid", ), ( EnumFilenames.ORGANIZATION_ADHOC, options.organization_adhoc_filename, OrganizationAdhoc, "orgid", ), ( EnumFilenames.AUTHORITY_ADHOC, options.authority_adhoc_filename, AuthorityAdhoc, "authorityid", ), ( EnumFilenames.AUTHORITY_ADHOC, options.authority_adhoc_filename, AuthorityAdhoc, "authorityid", ), ] self.file_data = self.populate_file_data(pattern_map_params) # TODO - revise comment def populate_file_data( self, pattern_map_params: List[Tuple[EnumFilenames, str, type, str]] ): """Populates the self.file_data field given a set of tuple params. pattern_map_params must adhere to this format: [ (file_type, filename, data_type, id_field), ] vars: file_type (EnumFilenames) -> The name of the dictionary. Defined as a value on EnumFilenames, such as EnumFilenames.AGENCY_ADHOC filename (str) -> The filepath of the given "file_type", such as migrationdata/test123.txt data_type (type) -> The type of data to be read at the location of the filename. For instance, each row of test123.txt may return data of type AgencyAdhoc id_field (str) -> Given the "data_type" of each row, this specifies what the "id" of that row is. For example, "agencyid". This is used so we can store each record in a dictionary rather than a list of values. return example: EnumFilenames.AUTHORITY_ADHOC: PatternMap( authority_adhoc_filename, self.strip_date_regex, AuthorityAdhoc, "authorityid", ), """ file_data = {} for file_type, filename, data_type, id_field in pattern_map_params: file_data[file_type] = PatternMap( filename, self.strip_date_regex, data_type, id_field, ) return file_data def parse_all_files(self, infer_filenames=True): """Clears all preexisting data then parses each related CSV file. infer_filenames: bool -> Determines if we should try to infer the filename if a default is passed in """ self.clear_file_data() for name, value in self.file_data.items(): is_domain_escrow = name == EnumFilenames.DOMAIN_ESCROW filename = f"{value.filename}" if filename in self.all_files_set: _file = f"{self.directory}{value.filename}" value.data = self.parse_csv_file( _file, self.seperator, value.data_type, value.id_field, is_domain_escrow, ) else: if not infer_filenames: logger.error(f"Could not find file: {filename}") continue # Infer filename logic # # This mode is used for internal development use and testing only. Rather than having # to manually define the filename each time, we can infer what the filename # actually is. # Not intended for use outside of that, as it is better to assume # the end-user wants to be specific. logger.warning(f"Attempting to infer filename: {filename}") for filename in self.all_files: default_name = name.value[1] match = value.try_infer_filename(filename, default_name) filename = match[0] can_infer = match[1] if can_infer: break if filename in self.all_files_set: logger.info(f"Infer success. Found file {filename}") _file = f"{self.directory}{filename}" value.data = self.parse_csv_file( _file, self.seperator, value.data_type, value.id_field, is_domain_escrow, ) continue # Log if we can't find the desired file logger.error(f"Could not find file: {filename}") def clear_file_data(self): for item in self.file_data.values(): file_type: PatternMap = item file_type.data = {} def parse_csv_file( self, file, seperator, dataclass_type, id_field, is_domain_escrow=False ): # Domain escrow is an edge case if is_domain_escrow: item_to_return = self._read_domain_escrow( file, seperator ) return item_to_return else: item_to_return = self._read_csv_file( file, seperator, dataclass_type, id_field ) return item_to_return # Domain escrow is an edgecase given that its structured differently data-wise. def _read_domain_escrow(self, file, seperator): dict_data = {} with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.reader(requested_file, delimiter=seperator) # clean the rows of any whitespace around delimiters for row in reader: yield (c.strip() for c in row) for row in reader: domain_name = row[0] date_format = "%Y-%m-%dT%H:%M:%SZ" # TODO - add error handling creation_date = datetime.strptime(row[7], date_format) expiration_date = datetime.strptime(row[11], date_format) dict_data[domain_name] = DomainEscrow( domain_name, creation_date, expiration_date ) return dict_data def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) dict_data = {} # clean the rows of any whitespace around delimiters for row in reader: yield (c.strip() for c in row) for row in reader: if None in row: logger.info("Skipping row with None key") logger.info(dataclass_type) for key, value in row.items(): logger.info(f"key: {key} value: {value}") TerminalHelper.prompt_for_execution(False, "COnintue?", "DEBUG") continue row_id = row[id_field] # To maintain pairity with the load_transition_domain # script, we store this data in lowercase. if id_field == "domainname" and row_id is not None: row_id = row_id.lower() dict_data[row_id] = dataclass_type(**row) # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} return dict_data