Skip to content

Commit

Permalink
Update system files setup for containers
Browse files Browse the repository at this point in the history 
The attribute provide_system_files creates a meta file in the
root tree named 'systemfiles'. The contents of this file were
produced by just a dump of the package database so far. For
a more generic use of this data some adaptions were needed.
First we allow to skip packages matching a pattern from being
part of the system files. Next we do not put ghost and doc
files into account. And last we handle library files in a different
file named 'systemfiles.libs' where we do not add symlink targets
if the target path is also part of the package. The consumer
of this information is flake-pilot which syncs that library system
files from the host via --copy-links. This allows a more generic
use with regards to versioned libraries e.g. libc
  • Loading branch information
@schaefi
schaefi committed Dec 13, 2024
1 parent 89959df commit b1da972
Show file tree
Hide file tree
Showing 10 changed files with 244 additions and 39 deletions.
2 changes: 1 addition & 1 deletion kiwi/schema/kiwi.rnc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3702,7 +3702,7 @@ div {
attribute type {
"bootstrap" | "delete" | "docker" | "image" |
"iso" | "oem" | "pxe" | "kis" | "oci" |
"uninstall"
"uninstall" | "systemfiles"
}
k.packages.profiles.attribute = k.profiles.attribute
k.packages.patternType.attribute =
Expand Down
1 change: 1 addition & 0 deletions kiwi/schema/kiwi.rng
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5587,6 +5587,7 @@ packages are only installed if this build type is requested.</a:documentation>
<value>kis</value>
<value>oci</value>
<value>uninstall</value>
<value>systemfiles</value>
</choice>
</attribute>
</define>
Expand Down
3 changes: 3 additions & 0 deletions kiwi/system/root_import/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,9 @@ def overlay_finalize(self, xml_state: XMLState) -> None:
with open(systemfiles, 'a') as systemfiles_fd:
# copy on write makes this file to become part of the delta
systemfiles_fd.write(os.linesep)
with open(f'{systemfiles}.libs', 'a') as systemlibs_fd:
# copy on write makes this file to become part of the delta
systemlibs_fd.write(os.linesep)

# Umount and rename upper to be the new root
self.overlay.umount()
Expand Down
95 changes: 80 additions & 15 deletions kiwi/system/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,9 @@ def export_modprobe_setup(self, target_root_dir: str) -> None:
options=['-a']
)

def export_package_file_list(self, target_dir: str, file_name: str) -> str:
def export_flake_pilot_system_file_list(
self, target_dir: str, file_name: str
) -> str:
"""
Export image package file list to the target_dir
and filename
Expand All @@ -638,7 +640,7 @@ def export_package_file_list(self, target_dir: str, file_name: str) -> str:
)
result_file = os.path.normpath(f'{target_dir}/{file_name}')
if packager == 'rpm':
self._export_rpm_file_list(result_file)
self._export_rpm_flake_pilot_system_file_list(result_file)
return result_file
return ''

Expand Down Expand Up @@ -829,10 +831,10 @@ def call_edit_boot_install_script(

def create_system_files(self) -> None:
"""
Create file list of packages
Create file list of packages to be used by flake-pilot
"""
if self.xml_state.build_type.get_provide_system_files():
self.export_package_file_list(
self.export_flake_pilot_system_file_list(
self.root_dir, Defaults.get_system_files_name()
)

Expand Down Expand Up @@ -1346,22 +1348,85 @@ def _text(self, section_content):
if section_content:
return section_content[0]

def _export_rpm_file_list(self, filename):
log.info('Export rpm packages file list')
def _export_rpm_flake_pilot_system_file_list(self, filename):
log.info('Export rpm packages system file list for flake-pilot')
dbpath_option = [
'--dbpath', self._get_rpm_database_location()
]
skip_list = self.xml_state.get_system_files_ignore_packages()
query_call = Command.run(
[
'rpm', '--root', self.root_dir,
'--noartifact', '--noghost', '-qal'
] + dbpath_option
['rpm', '--root', self.root_dir, '-qa'] + dbpath_option
)
with open(filename, 'w', encoding='utf-8') as packagefiles:
packagefiles.write(
os.linesep.join(sorted(query_call.output.splitlines()))
)
packagefiles.write(os.linesep)
libcheck = [
'/lib', '/usr/lib', '/lib64', '/usr/lib64'
]
packmeta = {}
for package in query_call.output.splitlines():
skip = False
for skip_package in skip_list:
if package.startswith(skip_package):
skip = True
break
if not skip and package:
package = package.strip()
packmeta[package] = {}
query_call = Command.run(
[
'rpm', '--root', self.root_dir, '-ql',
'--noghost', '--dump', package
] + dbpath_option
)
for raw in query_call.output.splitlines():
meta = raw.lstrip().split(' ')
if len(meta) == 11:
islib = False
for libpath in libcheck:
if meta[0].startswith(libpath):
islib = True
break
packmeta[package][meta[0]] = {
'islib': islib,
'isdoc': True if meta[8] == '1' else False,
'islink': True if meta[10] != 'X' else False,
'linkname': meta[10]
}

with open(filename, 'w', encoding='utf-8') as systemfiles:
for package in packmeta.keys():
for file in packmeta[package].keys():
meta = packmeta[package][file]
if not meta['isdoc'] and not meta['islib']:
systemfiles.write(f'{file}{os.linesep}')

with open(f'{filename}.libs', 'w', encoding='utf-8') as systemlibs:
for package in packmeta.keys():
for file in packmeta[package].keys():
meta = packmeta[package][file]
if meta['islib']:
if meta['islink']:
systemlibs.write(f'{file}{os.linesep}')
elif not self._has_link_target(packmeta, file):
systemlibs.write(f'{file}{os.linesep}')

def _has_link_target(
self, packages: Dict[str, Dict[str, Dict[str, str]]], filename: str
) -> bool:
if 'ld-linux' in filename:
# exceptional case. /lib.../ld-linux-* could be a
# symlink to /usr/bin/ld.so but only providing
# /usr/bin/ld.so is not enough to call programs.
# Thus in this case we indicate the file has no link
# target such that the later pilot sync takes it
# into account.
return False
for package in packages.keys():
for file in packages[package].keys():
meta = packages[package][file]
base_filename = os.path.basename(filename)
base_linkname = os.path.basename(meta['linkname'])
if meta['islink'] and base_filename == base_linkname:
return True
return False

def _export_rpm_package_list(self, filename):
log.info('Export rpm packages metadata')
Expand Down
11 changes: 11 additions & 0 deletions kiwi/xml_state.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,17 @@ def get_ignore_packages(self, section_type: str) -> List:
result.append(package.get_name().strip())
return sorted(result)

def get_system_files_ignore_packages(self) -> List[str]:
"""
List of ignore package names from the type="systemfiles"
packages section(s)
:return: package names
:rtype: list
"""
return self.get_ignore_packages('systemfiles')

def get_system_ignore_packages(self) -> List:
"""
List of ignore package names from the packages sections matching
Expand Down
5 changes: 5 additions & 0 deletions test/data/example_config.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,4 +260,9 @@
<packages type="uninstall">
<package name="shadow"/>
</packages>
<packages type="systemfiles">
<ignore name="zypp"/>
<ignore name="yast"/>
<ignore name="rpm"/>
</packages>
</image>
48 changes: 48 additions & 0 deletions test/data/rpm_ql_dump_glibc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/etc/bindresvport.blacklist 415 1733214801 4b0166e286cb27b577940432c6e39614b143b0d2c207dd3533906a19956e2c04 0100644 root root 1 0 0 X
/etc/gai.conf 0 1733214826 0000000000000000000000000000000000000000000000000000000000000000 0100644 root root 1 0 0 X
/etc/ld.so.cache 0 1733214801 0000000000000000000000000000000000000000000000000000000000000000 0100644 root root 1 0 0 X
/etc/ld.so.conf 206 1733214801 0ad7a03c5985fc18c3ee981325a60a7c3f8a169151b9cd653ddfe2e77f01fa26 0100644 root root 1 0 0 X
/etc/nsswitch.conf 2190 1733214801 6e084899135cda5df149d95e3dc79f22d1b4367b7c3b2fd74582d02be3c785cf 0100644 root root 1 0 0 X
/etc/rpc 1634 1733214698 3b24a975dcde688434258566813a83ce256a4c73efd7a8a9c3998327b0b4de68 0100644 root root 1 0 0 X
/lib64/ld-linux-x86-64.so.2 243504 1733214804 c70459f7af50cbae25fe31df8344062ee83b2963675e32ee103fd10a8a7be5df 0100755 root root 0 0 0 X
/lib64/ld-lsb-x86-64.so.3 20 1733214811 0000000000000000000000000000000000000000000000000000000000000000 0120777 root root 0 0 0 ld-linux-x86-64.so.2
/lib64/libBrokenLocale.so.1 7560 1733214804 ba61bdf8b15c60bc81a2fa080f22fb39126ace4029d7f6e6d3a483ff91a41129 0100755 root root 0 0 0 X
/lib64/libanl.so.1 7384 1733214804 d53507cacee2da3cdc82b9f4736f2acdd8b39d756f65a12bddc6fb8707d038b3 0100755 root root 0 0 0 X
/lib64/libc.so.6 2449760 1733214804 d51f437bdf319de3d2c37443316c98541987603308a834a5511330c6bb1c91b8 0100755 root root 0 0 0 X
/lib64/libc_malloc_debug.so.0 66640 1733214804 76901cfa755ad9fe4669982f311b1b417f720ce8984253a668e496d4b9400b49 0100755 root root 0 0 0 X
/lib64/libdl.so.2 7552 1733214804 1f2534a7db24829fdc8b32b6a16938b5a52efde17f1e175ded9a8cab2bde1b2c 0100755 root root 0 0 0 X
/lib64/libm.so.6 1037488 1733214804 f1e5e586ee981e8efec0eb149126118a3e4644502f939831ff1daaf474beb44c 0100755 root root 0 0 0 X
/lib64/libmvec.so.1 1076328 1733214804 730556bf17e863368aab990007f8b505563712ea2323a866779302c9bb92745f 0100755 root root 0 0 0 X
/lib64/libnss_compat.so.2 45672 1733214804 fe63514a1f16aef4587c81808310b2fc7d1a2c2180938839b3a5744e76276d4b 0100755 root root 0 0 0 X
/lib64/libnss_db.so.2 40864 1733214804 8c848c3462df5ab90a8d3d6e5576449949e5ad55631423b546c9f3097e0030d2 0100755 root root 0 0 0 X
/lib64/libnss_dns.so.2 7056 1733214804 6e1e6a5a863b549240168c3cf6542fd798d30a8c81c9e9fbb3baa7d82d809b4c 0100755 root root 0 0 0 X
/lib64/libnss_files.so.2 7056 1733214804 e4c4621713e92c81673c80850b041828a1aa6b9b6dcd6b1ef8cf20d95e63a743 0100755 root root 0 0 0 X
/lib64/libnss_hesiod.so.2 26680 1733214804 d8984863043b1a087d66bfd3f7a8f6a9d73b0df4144942b5cac4a0a4bbcd1af6 0100755 root root 0 0 0 X
/lib64/libpthread.so.0 8408 1733214804 4f9d2dca66e07c9206190f9b6dde9ddc6eb3de5883865f5a0f365590d3cb528b 0100755 root root 0 0 0 X
/lib64/libresolv.so.2 73640 1733214804 d9c7d53ab627c3f116e2520c6bd29fa4403295ba02af73d95430e38fb9ba1876 0100755 root root 0 0 0 X
/lib64/librt.so.1 8176 1733214804 84bf7cec9a66333c1d33a93bd9d8f2d7211e25c8a4f328aa2739983c5b84da0b 0100755 root root 0 0 0 X
/lib64/libthread_db.so.1 49216 1733214804 5a0a2cf166b6207d905fc6f3e4d5dad6774d9bc76be81a58e28a6ef6ad51971a 0100755 root root 0 0 0 X
/lib64/libutil.so.1 7392 1733214804 ab7afc07bdc5be89ff703dc808b43c6dbc3c4885848b9463d5c5b7d243fac8a3 0100755 root root 0 0 0 X
/sbin/ldconfig 1073856 1733214804 86143be7fb0e7619d9eb8153591491f2e3ab821a9cdb44483954eb6fdf443f23 0100755 root root 0 0 0 X
/usr/bin/gencat 27120 1733214804 9754597fd4521fd9b2392fb91fbaea756d81a6bd9fe4d39929db38f9719c5aef 0100755 root root 0 0 0 X
/usr/bin/getconf 22 1733214811 0000000000000000000000000000000000000000000000000000000000000000 0120777 root root 0 0 0 ../lib/getconf/getconf
/usr/bin/getent 31616 1733214804 715f8eaf7d145853008a8169a4fb42ffa5a182288c609b827d9d2e9d5ca14872 0100755 root root 0 0 0 X
/usr/bin/iconv 64432 1733214804 a645cdd3ce6fa2c10ee097db9866cc199d8f3d15095c312a8a170b7acd21c9ef 0100755 root root 0 0 0 X
/usr/bin/ld.so 27 1733214811 0000000000000000000000000000000000000000000000000000000000000000 0120777 root root 0 0 0 /lib64/ld-linux-x86-64.so.2
/usr/bin/ldd 5372 1733214700 d89312068d0940fc9ae75eac6f81ff60ca6750318567c4dd0caa646d966f6cd9 0100755 root root 0 0 0 X
/usr/bin/locale 51112 1733214804 9e1cc2506dc7aaa94db572320331481f92be2c6dad78a7cfc46b7988e2c6d847 0100755 root root 0 0 0 X
/usr/bin/localedef 336352 1733214804 fcee441a70cb59c6c709838582f6fc2ff015dfa4f4350d8a1f117847a9f8502e 0100755 root root 0 0 0 X
/usr/lib/getconf 0 1733214804 0000000000000000000000000000000000000000000000000000000000000000 040755 root root 0 0 0 X
/usr/lib/getconf/POSIX_V6_LP64_OFF64 31064 1733214804 347bece701b32a42b6cc545e9f63640765cfb493b9cc1c68507483a53c2db984 0100755 root root 0 0 0 X
/usr/lib/getconf/POSIX_V7_LP64_OFF64 31064 1733214804 347bece701b32a42b6cc545e9f63640765cfb493b9cc1c68507483a53c2db984 0100755 root root 0 0 0 X
/usr/lib/getconf/XBS5_LP64_OFF64 31064 1733214804 347bece701b32a42b6cc545e9f63640765cfb493b9cc1c68507483a53c2db984 0100755 root root 0 0 0 X
/usr/lib/getconf/getconf 31064 1733214804 347bece701b32a42b6cc545e9f63640765cfb493b9cc1c68507483a53c2db984 0100755 root root 0 0 0 X
/usr/sbin/iconvconfig 31376 1733214805 4cde0a5234ffd3caf7907f35a5e4b3de08eab17115bedd062cca4d19c17d2464 0100755 root root 0 0 0 X
/usr/share/doc/packages/glibc 0 1733214826 0000000000000000000000000000000000000000000000000000000000000000 040755 root root 0 0 0 X
/usr/share/doc/packages/glibc/gai.conf 2584 1690826056 76a5771adee7b9f36c7ae66eae78d72f325557500269107f2d98a7e3560a1808 0100644 root root 0 1 0 X
/usr/share/licenses/glibc 0 1733214826 0000000000000000000000000000000000000000000000000000000000000000 040755 root root 0 0 0 X
/usr/share/licenses/glibc/LICENSES 18943 1690826056 b33d0bd9f685b46853548814893a6135e74430d12f6d94ab3eba42fc591f83bc 0100644 root root 0 0 0 X
/usr/share/man/man1/gencat.1.gz 2619 1733214801 2aa75ddeb08e1772018ed74fe8bc1f1c4467c49c0a939052a2fe6ce2211f07da 0100644 root root 0 1 0 X
/usr/share/man/man1/getconf.1.gz 2346 1733214801 ec3d91ae40cbbf7aafb047a4799c25e00a04d340c0febe83d72f66ca2dfc7dc0 0100644 root root 0 1 0 X
/usr/share/man/man5/locale.alias.5.gz 949 1733214801 397f8cd51dc51c12e1b387201c40191e72bc5b9a92ffa77a0864505e5bc0ec86 0100644 root root 0 1 0 X
/var/cache/ldconfig 0 1733214801 0000000000000000000000000000000000000000000000000000000000000000 040700 root root 0 0 0 X
1 change: 1 addition & 0 deletions test/unit/system/root_import/base_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ def test_overlay_finalize(
]
# create removed files metadata for later host provisioning
assert file_handle.write.call_args_list == [
call('\n'),
call('\n'),
call('/file_a'), call(os.linesep),
call('/file_b'), call(os.linesep)
Expand Down
Loading

0 comments on commit b1da972

Please sign in to comment.