o f@sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZmZmZmZmZddlmZmZmZmZmZmZddlmZddlmZddlmZdd lm Z dd l!m"Z"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)ddl*m+Z+dZ,dZ-dZ.dZ/e,e-e.gZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8e9e:Z;dddfdddfd d!dfd"ZGd'd(d(e?Z@Gd)d*d*e?ZAGd+d,d,e?ZBdYd/d0ZCe5fd1d2ZDed3gd4ZEed5d6d7gZFGd8dde)ejGd9ZHd:d;ZIdZKd?d@ZL AdZdBeJd?r+zaws-govcCr$r%r'r(r'r'r*r+@r,z azure-chinacCr$)Nazurer'r(r'r'r*r+Ar,)zcn-zus-gov-chinac@s2eZdZdZdZdZdZdZdZde fdd Z d S) NetworkConfigSourcezb Represents the canonical list of network config sources that cloud-init knows about. cmdlineds system_cfgfallback initramfsreturncC|jSNvalueselfr'r'r*__str__RzNetworkConfigSource.__str__N) __name__ __module__ __qualname____doc__CMD_LINEDS SYSTEM_CFGFALLBACK INITRAMFSstrr<r'r'r'r*r/Esr/c@s&eZdZdZdZdZdefddZdS)NicOrderzRepresents ways to sort NICsmacnic_namer5cCr6r7r8r:r'r'r*r<\r=zNicOrder.__str__N)r>r?r@rAMACNIC_NAMErGr<r'r'r'r*rHVs rHc@eZdZdZdS)DatasourceUnpickleUserDataErrorzERaised when userdata is unable to be unpickled due to python upgradesNr>r?r@rAr'r'r'r*rN`rNc@ eZdZdS)DataSourceNotFoundExceptionNr>r?r@r'r'r'r*rRdrRc@rM)InvalidMetaDataExceptionz8Raised when metadata is broken, unavailable or disabled.NrOr'r'r'r*rUhrPrUr'c Cst|}g}g}|D]V\}}|r|d|}n|}||vs(||vr-||t|trD|drD|||dd||<t|t rct |||} | | d| | d| ||<q t ||d<t ||d<|S)zProcess all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @return Dict copy of processed metadata. /zci-b64:rVbase64_encoded_keyssensitive_keys)copydeepcopyitemslowerappend isinstancerG startswithreplacedictprocess_instance_metadataextendpopsorted) metadatakey_pathrYmd_copyrX sens_keyskeyval sub_key_path return_valr'r'r*rcls0        rccCs|dgs|St|}|dD]+}|d}|}|D]}||vr4t||tr4||dkr4||}q||vr=|||<q|S)zRedact any sensitive keys from to provided metadata dictionary. Replace any keys values listed in 'sensitive_keys' with redact_value. rYrW)getrZr[splitr_rb)rg redact_valuerirh path_partsobjpathr'r'r*redact_sensitive_keyss"     rv URLParams)max_wait_secondstimeout_seconds num_retriessec_between_retriesDataSourceHostnamehostname is_defaultc @seZdZUeZdZdZdZdZdZ dZ e e e efed<ejejejejfZeedfed<dZdZd Zd Zejejejej ej!hiZ"ejejhiZ#d e$fd e$fd ifddddddf Z%eeee&fdfed<dZ'dZ(eedfed<dZ)dZ*e eed<d Z+dxde,de-fddZ.de/ddfdd Z0d!d"Z1de2fd#d$Z3de2fd%d&Z4d'd(Z5d)d*Z6dyd,d-Z7de2fd.d/Z8dzd1d2Z9de2fd3d4Z:d5d6Z;d{d7d8Ze?d=d>Z@e?d?d@ZAdAdBZBe?dCdDZCdEdFZDe?dGdHZEdIdJZFdKdLZGdMdNZHdOdPZIdQdRZJdSdTZKdUdVZLdWdXZMdYdZZNd[d\ZOe?d]d^ZPe?d_d`ZQdadbZRd|dcddZSdedfZTdgeUefdhdiZVdgeUede2fdjdkZWdldmZXdndoZYeZd}dpdqZ[e?drdsZ\dtduZ]dvdwZ^dS)~r z en_US.UTF-8_undefN_crawled_metadata.network_config_sourcesro  ec2_metadata network_jsonrg)userdataN) userdata_rawN) vendordataN)vendordata_rawN) vendordata2N)vendordata2_rawNcached_attr_defaultsF) combined_cloud_config merged_cfgmerged_system_cfgzsecurity-credentialsr user-datarr vendor-datazds/vendor_datasensitive_metadata_keysextra_hotplug_udev_rulesdistropathscCs||_||_||_d|_i|_d|_d|_d|_d|_d|_ d|_ t |_ t |_ t|jd|jfi|_|js9i|_|sDt|j|_dS||_dS)N datasource)sys_cfgrrrrgrrrrrmetadata_addressUNSETrrrget_cfg_by_pathdsnameds_cfgudUserDataProcessorud_proc)r;rrrrr'r'r*__init__<s* zDataSource.__init__ci_pkl_versionr5c Csdddtddtdddd }|D]\}}t||s t|||qt|ds.t|dddt|drW|jdurYzt|jWdStyV}z td|t |d}~wwdSdS) z(Perform deserialization fixes for Paths.NF) r_platform_type _subplatformrrrrskip_hotplug_detectrrcheck_if_fallback_is_allowedcSdSNFr'r'r'r'r*r+iz&DataSource._unpickle..rz:Unable to unpickle datasource: %s. Ignoring current cache.) rr\hasattrsetattrrrGAttributeErrorLOGdebugrN)r;rexpected_attrsrkr9er'r'r* _unpickleVs:   zDataSource._unpicklecCs t|Sr7robj_namer:r'r'r*r<zs zDataSource.__str__cCr)z#Check if running on this datasourceTr'r:r'r'r* ds_detect}zDataSource.ds_detectcCsN|jtkrtd|dS|jdg|jgkr%td|dSdS)aJOverride if either: - only a single datasource defined (nothing to fall back to) - command line argument is used (ci.ds=OpenStack) Note: get_cmdline() is required for the general case - when ds-identify does not run, _something_ needs to detect the kernel command line definition. z6Kernel command line set to use a single datasource %s.Tdatasource_listz2Datasource list set to use a single datasource %s.F)rr] parse_cmdlinerrrrpr:r'r'r*override_ds_detects zDataSource.override_ds_detectcCs<|r|S|rtd||Std|dS)z&Overrides runtime datasource detectionz Detected %szDid not detect %sF)r _get_datarrrr:r'r'r*_check_and_get_datas zDataSource._check_and_get_datacCs|j}|}|j}|d}diddgd|d|dt|j|j|jd|jd |jd |d d d |d dd|d dd|jd|d|dd|d|d|ddd|d||dd|j|j |d|ddiS)z2Return a dictionary of standardized metadata keys.sys_infov1 _beta_keys subplatformavailability-zoneavailability_zonecloud_idr! cloud_namerdistrdistro_versionrdistro_releaseplatformpublic_ssh_keyspython_versionpython instance-id instance_idkernel_releaseunamelocal-hostnamelocal_hostnamevariant)machineregionrsystem_platformr) get_hostnamer}get_instance_idrcanonical_cloud_idrr platform_typeget_public_ssh_keysr)r; instance_datarrrsysinfor'r'r*_get_standardized_metadatas`         z%DataSource._get_standardized_metadatar'cCsP|jsdS|r |}n|j}|D]\}}t||rt|||q|s&d|_dSdS)zReset any cached metadata attributes to datasource defaults. @param attr_defaults: Optional tuple of (attr, value) pairs to set instead of cached_attr_defaults. NF) _dirty_cacherrr)r; attr_defaults attr_values attributer9r'r'r*clear_cached_attrss    zDataSource.clear_cached_attrscCs"d|_|}|s |S||S)zDatasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. T)rrpersist_instance_data)r; return_valuer'r'r*get_datas zDataSource.get_dataTc CsX|rtj|jjrt||jd|jdur/t |j}| dd| ddd|i}ndd|j ii}|j t krB|j |dd<|jt krN|j|dd<t|dd <t |j|d <d |d d <t |d |d <d |d d <t|d<|||zt|}tt||jd}Wn5ty}ztdt|WYd}~dSd}~wty}ztdt|WYd}~dSd}~ww|j d}|d!dd}tj"|jj#d} t$| d||dd} | d|} tj%| rtj&| } tj'| | dd| r| | krt(| t)||dd|j d} t)| t*|dS)aPProcess and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. :param write_cache: boolean set True to persist obj.pkl when instance_link exists. @return True on successful write, False otherwise. obj_pklNrrr1 meta_datarr_docrz, (str, bool). is_default is a bool and it's true only if hostname is localhost and was returned by util.get_hostname() as a default. This is used to differentiate with a user-defined localhost hostname. Optionally return (None, False) when metadata_only is True and local-hostname data is not available. localdomain localhostFrNT.rzip-%srrz%s.%s)rgrpr|rrget_fqdn_from_hostsfindrGrqris_ipv4_address gethostbyaddrrar=r) r;fqdn resolve_ip metadata_only defdomaindefhostdomainr~toksr} hosts_fqdnlhostr'r'r*r,sF            zDataSource.get_hostnamecCs|jj|dS)N) data_source)rget_package_mirror_infor:r'r'r*r_tsz"DataSource.get_package_mirror_infosource_event_typescCsPi}|D]!}|jD]\}}||vr$||st||<|||q q|Sr7)supported_update_eventsr\rpsetadd)r;r`supported_eventsevent update_scope update_eventsr'r'r*get_supported_eventsws  zDataSource.get_supported_eventsc Cs||}|D]\}}td|jddd|D|d|tffq |r6||}|r6dStd|ddd|Dd S) aRefresh cached metadata if the datasource supports this event. The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @param source_event_types: List of EventTypes which may trigger a metadata update. @return True if the datasource did successfully update cached metadata due to source_event_type. z:Update datasource metadata and %s config due to events: %s, cSg|]}|jqSr'r8.0rer'r'r* z;DataSource.update_metadata_if_supported..z _%s_configTz(Datasource %s not updated for events: %scSrjr'r8rkr'r'r*rmrnF) rhr\rrr9rrrr)r;r`rdscopematched_eventsresultr'r'r*update_metadata_if_supporteds& z'DataSource.update_metadata_if_supportedcCrrr')r;rr'r'r*check_instance_idrzDataSource.check_instance_idcCr)acheck_if_fallback_is_allowed() Checks if a cached ds is allowed to be restored when no valid ds is found in local mode by checking instance-id and searching valid data through ds list. @return True if a ds allows fallback, False otherwise. Fr'r:r'r'r*rsz'DataSource.check_if_fallback_is_allowedcCsT|durt}|dur t}|D]}|durq||vr|Std|||S|S)Nz%invalid dsmode '%s', using default=%s)DSMODE_NETWORK VALID_DSMODESrr) candidatesdefaultvalid candidater'r'r*_determine_dsmodeszDataSource._determine_dsmodecCrEr7r'r:r'r'r*network_configrzDataSource.network_configcCr)a(setup(is_new_instance) This is called before user-data and vendor-data have been processed. Unless the datasource has set mode to 'local', then networking per 'fallback' or per 'network_config' will have been written and brought up the OS at this point. Nr')r;is_new_instancer'r'r*setups zDataSource.setupcCr)aactivate(cfg, is_new_instance) This is called before the init_modules will be called but after the user-data and vendor-data have been fully processed. The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. is_new_instance is a boolean indicating if this is a new instance. Nr')r;cfgr|r'r'r*activates zDataSource.activater7)r'T)F)FFF)NN)_r>r?r@rtdsmoderHrr*rrrrr rrG__annotations__r/rBrFrDrCrr rrrrrrrBOOT_NEW_INSTANCEBOOT BOOT_LEGACYHOTPLUGradefault_update_eventsrrrrrrr_ci_pkl_versionrrrrrr<boolrrrrrrrrrr r#r&propertyrrr'rr,rrrr"r%r5rr8rDrGrIrrrrr_rrhrrrsr staticmethodrzr{r}rr'r'r'r*r s     $ & L <         H  &   ) metaclasscCsg}|s|St|tr|St|ttfrt|St|trC|D]\}}t|tr/|g}t|ttfrB|D] }|rA||q8q#|Sr7)r_rG splitlineslistrbrbr\r^) pubkey_datakeys_keynameklistpkeyr'r'r*r6s"    r6r5c Cs.t|||}dd|D}t|vrdnd} td| |t||D]j\} } tjd| ddd | | fd | | f|d } z?| 3td | | |||} | t j grkd | | f| _ | t | fWdWSWdn1suwYWq!tyttd| Yq!wdd|}t|)NcSsg|]}t|qSr'r)rlr4r'r'r*rm szfind_source..networkrz#Searching for %s data source in: %sz search-%sr rVzsearching for %s data from %szno %s data found from %s)name descriptionmessageparentz%Seeing if we can get any data from %szfound %s data from %szGetting data from %s failedz4Did not find any data source, searched classes: (%s)ri) list_sources DEP_NETWORKrrziprReportEventStackrarrrrrrrrrrrrR)rrrds_depscfg_listpkg_listreporterds_listds_namesrrclsmyrepsmsgr'r'r* find_source s>       rc Csg}td||||D]5}t|}t||dg\}}|s%td||D]}t|} t| d} | |} | r@|| nq'q |S)zReturn a list of classes that have the same depends as 'depends' iterate through cfg_list, loading "DataSource*" modules and calling their "get_datasource_list". Return an ordered list of classes that match (if any) zLLooking for data source in: %s, via packages %s that matches dependencies %sget_datasource_listzDCould not import %s. Does the DataSource exist and is it importable?) rrr "match_case_insensitive_module_name find_moduleerror import_modulegetattrrd) rdependsrsrc_listr1ds_namem_locs _looked_locsm_locmodlistermatchesr'r'r*r*s6    r system-uuidfieldcCs*|sdSt|}|s dS||kSr)r read_dmi_datar])rr dmi_valuer'r'r*instance_id_matches_system_uuidNs  rcCsl|st}|st}|tkr|tkr|S|StD]\}}|\}}||r-||r-|Sq|tkr4|S|S)z@Lookup the canonical cloud-id for a given cloud_name and region.)r)CLOUD_ID_REGION_PREFIX_MAPr\r`)rrrprefix cloud_id_testr valid_cloudr'r'r*r\s rTcCsf|sdSt|tr |St|trt|St|tr+|dur't|dddStdtdt |)aLdata: a loaded object (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: None: return None string: return string list: return data the list is then processed in UserDataProcessor dict: return convert_vendordata(data.get('cloud-init')) NTz cloud-initF)recursez'vendordata['cloud-init'] cannot be dictz$Unknown data type for vendordata: %s) r_rGrrZr[rbconvert_vendordatarprr-)datarr'r'r*ros     rc@rQ)BrokenMetadataNrSr'r'r'r*rrTrcCs4g}t|}|D]\}}|t|kr||q|Sr7)rbr^)rrret_listdepsetrdepsr'r'r*list_from_dependss   rrtfnamecCsnzt|}Wntyttd|YdSwz tj||dddWdSty6ttd|YdSw)z[Use pickle to serialize Datasource to a file as a cache. :return: True on success zFailed pickling datasource %sFwb)omoderz Failed pickling datasource to %sT)pickledumpsrrrrr)rtr pk_contentsr'r'r*rs  rc Csd}zt|}Wnty(}ztj|rtd||WYd}~nd}~ww|s-dSzt |WSt y=YdStyMt td|YdSw)zBUse pickle to deserialize a instance Datasource from a cache file.Nzfailed loading pickle in %s: %sz#Failed loading pickled blob from %s) rload_binary_filerrruisfilerrrrrNr)rpickle_contentsrr'r'r*pkl_loads$    rcCs ttS)zCheck if command line argument for this datasource was passed Passing by command line overrides runtime datasource detection )parse_cmdline_or_dmir get_cmdliner'r'r'r*rs rinputcCstd|}td|}td|}|p|p|}|p|}|r7|d}tjd|d|dd|d d |rC|drC|dSd S) Nz(?:^|\s)ds=([^\s;]+)z(?:^|\s)ci\.ds=([^\s;]+)z (?:^|\s)ci\.datasource=([^\s;]+)rz8Defining the datasource on the command line using ci.ds=z or ci.datasource=z23.2zUse ds=z instead) deprecateddeprecated_version extra_messagerV)researchgroupstripr deprecate)r ds_parse_0 ds_parse_1 ds_parse_2r1rrr'r'r*rs&      r)rVr')rr)XabcrZrloggingrrr collectionsrenumrrtypingrrrrr r cloudinitr r r rrrrrrcloudinit.atomic_helperrcloudinit.distrosrcloudinit.eventrrcloudinit.filtersrcloudinit.helpersrcloudinit.persistencercloudinit.reportingrDSMODE_DISABLED DSMODE_LOCALrt DSMODE_PASSruDEP_FILESYSTEMr DS_PREFIXrREDACT_SENSITIVE_VALUEr+rr) getLoggerr>rrr/rHrrNrRrUrcrvrwr|ABCMetar r6rGrrrrrrIOErrorrrrrrrr'r'r'r*s                    # 6  !%