libzypp: zypp/ng/repo/workflows/susetags.cc Source File

libzypp 17.38.14

susetags.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
9#include "susetags.h"
10#include "zypp-core/base/Regex.h"
11#include <zypp-core/ng/ui/ProgressObserver>
12#include <zypp-media/ng/ProvideSpec>
13#include <zypp/ng/Context>
14#include <zypp/ZConfig.h>
15
16#include <zypp-core/parser/ParseException>
17
18#include <zypp/parser/susetags/ContentFileReader.h>
19#include <zypp/parser/susetags/RepoIndex.h>
20
21
22#include <zypp/ng/repo/workflows/repodownloaderwf.h>
23#include <zypp/ng/workflows/checksumwf.h>
24
25#undef ZYPP_BASE_LOGGER_LOGGROUP
26#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::repomanager"
27
28 namespace zyppng::SuseTagsWorkflows {
29
30 namespace {
31
32 using namespace zyppng::operators;
33
39 struct StatusLogic {
40 public:
41 using ProvideType = typename Context::ProvideType;
42 using MediaHandle = typename ProvideType::MediaHandle;
43 using ProvideRes = typename ProvideType::Res;
44
45 StatusLogic( repo::DownloadContextRef ctx, MediaHandle &&media )
46 : _ctx(std::move(ctx))
47 , _handle(std::move(media))
48 {}
49
50 MaybeAwaitable<expected<zypp::RepoStatus>> execute() {
51 return _ctx->zyppContext()->provider()->provide( _handle, _ctx->repoInfo().path() / "content" , ProvideFileSpec().setMirrorsAllowed(false) )
52 | [this]( expected<ProvideRes> contentFile ) {
53
54 // mandatory master index is missing -> stay empty
55 if ( !contentFile )
56 return makeReadyTask( make_expected_success (zypp::RepoStatus() ));
57
58 zypp::RepoStatus status ( contentFile->file() );
59
60 if ( !status.empty() /* && _ctx->repoInfo().requireStatusWithMediaFile() */ ) {
61 return _ctx->zyppContext()->provider()->provide( _handle, "/media.1/media" , ProvideFileSpec().setMirrorsAllowed(false) )
62 | [status = std::move(status)]( expected<ProvideRes> mediaFile ) mutable {
63 if ( mediaFile ) {
64 return make_expected_success(status && zypp::RepoStatus( mediaFile->file()) );
65 }
66 return make_expected_success( std::move(status) );
67 };
68 }
69 return makeReadyTask( make_expected_success(std::move(status)) );
70 };
71 }
72
73 repo::DownloadContextRef _ctx;
74 MediaHandle _handle;
75 };
76 }
77
78 MaybeAwaitable<expected<zypp::RepoStatus> > repoStatus(repo::DownloadContextRef dl, ProvideMediaHandle mediaHandle)
79 {
80 StatusLogic impl( std::move(dl), std::move(mediaHandle) );
81 zypp_co_return zypp_co_await( impl.execute() );
82 }
83
84 namespace {
85
86 using namespace zyppng::operators;
87
88 // search old repository file file to run the delta algorithm on
89 static zypp::Pathname search_deltafile( const zypp::Pathname &dir, const zypp::Pathname &file )
90 {
91 zypp::Pathname deltafile(dir + file.basename());
92 if (zypp::PathInfo(deltafile).isExist())
93 return deltafile;
94 return zypp::Pathname();
95 }
96
97 struct DlLogic {
98 public:
99
100 using ProvideType = typename Context::ProvideType;
101 using MediaHandle = typename ProvideType::MediaHandle;
102 using ProvideRes = typename ProvideType::Res;
103
104 DlLogic( repo::DownloadContextRef ctx, MediaHandle &&mediaHandle, ProgressObserverRef &&progressObserver )
105 : _ctx( std::move(ctx))
106 , _mediaHandle(std::move(mediaHandle))
107 , _progressObserver(std::move(progressObserver))
108 {}
109
110 auto execute() {
111 // download media file here
112 return RepoDownloaderWorkflow::downloadMediaInfo( _mediaHandle, _ctx->destDir() )
113 | [this]( expected<zypp::ManagedFile> &&mediaInfo ) {
114
115 // remember the media info if we had one
116 if ( mediaInfo ) _ctx->files().push_back ( std::move(mediaInfo.get()) );
117
118 if ( _progressObserver ) _progressObserver->inc();
119
120 return RepoDownloaderWorkflow::downloadMasterIndex( _ctx, _mediaHandle, _ctx->repoInfo().path() / "content" )
121 | inspect( incProgress( _progressObserver ) )
122 | and_then( [this] ( repo::DownloadContextRef && ) {
123
124 zypp::Pathname contentPath = _ctx->files().front();
125 std::vector<zypp::OnMediaLocation> requiredFiles;
126
127 // Content file first to get the repoindex
128 try {
129 const zypp::Pathname &inputfile = contentPath;
130 zypp::parser::susetags::ContentFileReader content;
131 content.setRepoIndexConsumer( [this]( const auto & data_r ) {
132 MIL << "Consuming repo index" << std::endl;
133 _repoindex = data_r;
134 });
135 content.parse( inputfile );
136
137 if ( ! _repoindex ) {
138 ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No repository index in content file." ) );
139 }
140
141 MIL << "RepoIndex: " << _repoindex << std::endl;
142 if ( _repoindex->metaFileChecksums.empty() ) {
143 ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No metadata checksums in content file." ) );
144 }
145
146 if ( _repoindex->signingKeys.empty() ) {
147 WAR << "No signing keys defined." << std::endl;
148 }
149
150 // Prepare parsing
151 zypp::Pathname descr_dir = _repoindex->descrdir; // path below reporoot
152 //_datadir = _repoIndex->datadir; // path below reporoot
153
154 std::map<std::string,zypp::parser::susetags::RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
155
156 for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
157 {
158 // omit unwanted translations
159 if ( zypp::str::hasPrefix( it->first, "packages" ) )
160 {
161 static const zypp::str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
162 zypp::str::smatch what;
163
164 if ( zypp::str::regex_match( it->first, what, rx_packages ) ) {
165 if ( what[4].empty() // packages(.gz)?
166 || what[4] == "DU"
167 || what[4] == "en" )
168 { ; /* always downloaded */ }
169 else if ( what[4] == "FL" )
170 { continue; /* never downloaded */ }
171 else
172 {
173 // remember and decide later
174 availablePackageTranslations[what[4]] = it;
175 continue;
176 }
177 }
178 else
179 continue; // discard
180
181 } else if ( it->first == "patterns.pat"
182 || it->first == "patterns.pat.gz" ) {
183 // take all patterns in one go
184
185 } else if ( zypp::str::endsWith( it->first, ".pat" )
186 || zypp::str::endsWith( it->first, ".pat.gz" ) ) {
187
188 // *** see also zypp/parser/susetags/RepoParser.cc ***
189
190 // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
191 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
192 // split at dots, take .pat or .pat.gz into account
193
194 std::vector<std::string> patparts;
195 unsigned archpos = 2;
196 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
197 unsigned count = zypp::str::split( it->first, std::back_inserter(patparts), "." );
198 if ( patparts[count-1] == "gz" )
199 archpos++;
200
201 if ( count > archpos ) {
202 try { // might by an invalid architecture
203 zypp::Arch patarch( patparts[count-archpos] );
204 if ( !patarch.compatibleWith( zConfig().systemArchitecture() ) ) {
205 // discard, if not compatible
206 MIL << "Discarding pattern " << it->first << std::endl;
207 continue;
208 }
209
210 } catch ( const zypp::Exception & excpt ) {
211 WAR << "Pattern file name does not contain recognizable architecture: " << it->first << std::endl;
212 // keep .pat file if it doesn't contain an recognizable arch
213 }
214 }
215 }
216
217 MIL << "adding job " << it->first << std::endl;
218 auto location = zypp::OnMediaLocation( repoInfo().path() + descr_dir + it->first, 1 )
219 .setChecksum( it->second )
220 .setDeltafile( search_deltafile( _ctx->deltaDir() + descr_dir, it->first) );
221
222 requiredFiles.push_back( std::move(location) );
223 }
224
225
226 // check whether to download more package translations:
227 {
228 auto fnc_checkTransaltions( [&]( const zypp::Locale & locale_r ) {
229 for ( zypp::Locale toGet( locale_r ); toGet; toGet = toGet.fallback() ) {
230 auto it( availablePackageTranslations.find( toGet.code() ) );
231 if ( it != availablePackageTranslations.end() ) {
232 auto mit( it->second );
233 MIL << "adding job " << mit->first << std::endl;
234 requiredFiles.push_back( zypp::OnMediaLocation( repoInfo().path() + descr_dir + mit->first, 1 )
235 .setChecksum( mit->second )
236 .setDeltafile( search_deltafile( deltaDir() + descr_dir, mit->first) ));
237 break;
238 }
239 }
240 });
241
242 for ( const zypp::Locale & it : zConfig().repoRefreshLocales() ) {
243 fnc_checkTransaltions( it );
244 }
245 fnc_checkTransaltions( zConfig().textLocale() );
246 }
247
248 for( const auto &it : _repoindex->mediaFileChecksums ) {
249 // Repo adopts license files listed in HASH
250 if ( it.first != "license.tar.gz" )
251 continue;
252
253 MIL << "adding job " << it.first << std::endl;
254 requiredFiles.push_back( zypp::OnMediaLocation ( repoInfo().path() + it.first, 1 )
255 .setChecksum( it.second )
256 .setDeltafile( search_deltafile( deltaDir(), it.first ) ));
257 }
258
259 for( const auto &it : _repoindex->signingKeys ) {
260 MIL << "adding job " << it.first << std::endl;
261 zypp::OnMediaLocation location( repoInfo().path() + it.first, 1 );
262 location.setChecksum( it.second );
263 requiredFiles.push_back( std::move(location) );
264 }
265
266 } catch ( const zypp::Exception &e ) {
267 ZYPP_CAUGHT( e );
268 return makeReadyTask(expected<repo::DownloadContextRef>::error( ZYPP_FWD_CURRENT_EXCPT() ) );
269 } catch ( ... ) {
270 return makeReadyTask(expected<repo::DownloadContextRef>::error( ZYPP_FWD_CURRENT_EXCPT() ) );
271 }
272
273 // add the required files to the base steps
274 if ( _progressObserver ) _progressObserver->setBaseSteps ( _progressObserver->baseSteps () + requiredFiles.size() );
275
276 return transform_collect ( std::move(requiredFiles), [this]( zypp::OnMediaLocation file ) {
277
278 return DownloadWorkflow::provideToCacheDir( _ctx, _mediaHandle, file.filename(), ProvideFileSpec(file) )
279 | inspect ( incProgress( _progressObserver ) );
280
281 }) | and_then ( [this]( std::vector<zypp::ManagedFile> &&dlFiles ) {
282 auto &downloadedFiles = _ctx->files();
283 downloadedFiles.insert( downloadedFiles.end(), std::make_move_iterator(dlFiles.begin()), std::make_move_iterator(dlFiles.end()) );
284 return expected<repo::DownloadContextRef>::success( std::move(_ctx) );
285 });
286 });
287
288 } | finishProgress( _progressObserver );
289 }
290
291 private:
292
293 const zypp::RepoInfo &repoInfo() const {
294 return _ctx->repoInfo();
295 }
296
297 const zypp::filesystem::Pathname &deltaDir() const {
298 return _ctx->deltaDir();
299 }
300
301 zypp::ZConfig &zConfig() {
302 return _ctx->zyppContext()->config();
303 }
304
305 repo::DownloadContextRef _ctx;
306 zypp::parser::susetags::RepoIndex_Ptr _repoindex;
307 MediaHandle _mediaHandle;
308 ProgressObserverRef _progressObserver;
309 };
310 }
311
312 MaybeAwaitable<expected<repo::DownloadContextRef> > download(repo::DownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
313 {
314 DlLogic impl( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
315 zypp_co_return zypp_co_await( impl.execute() );
316 }
317
318}
ContentFileReader.h
for_
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition Easy.h:27
ZYPP_CAUGHT
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition Exception.h:475
ZYPP_FWD_CURRENT_EXCPT
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition Exception.h:471
ZYPP_THROW
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:459
MIL
#define MIL
Definition Logger.h:103
WAR
#define WAR
Definition Logger.h:104
Regex.h
RepoIndex.h
ZConfig.h
checksumwf.h
zypp::OnMediaLocation::filename
const Pathname & filename() const
The path to the resource on the medium.
Definition onmedialocation.cc:90
zypp::RepoStatus
Track changing files or directories.
Definition RepoStatus.h:41
zypp::RepoStatus::empty
bool empty() const
Whether the status is empty (empty checksum).
Definition RepoStatus.cc:234
zypp::filesystem::PathInfo
Wrapper class for stat/lstat.
Definition PathInfo.h:226
zypp::filesystem::Pathname
Pathname.
Definition Pathname.h:48
zypp::filesystem::Pathname::basename
std::string basename() const
Return the last component of this path.
Definition Pathname.h:137
zypp::parser::susetags::ContentFileReader::parse
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
Definition ContentFileReader.cc:215
zypp::parser::susetags::ContentFileReader::setRepoIndexConsumer
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
Definition ContentFileReader.h:66
zyppng::Context::ProvideType
Provide ProvideType
Definition context.h:34
zyppng::ProvideFileSpec
Definition providespec.h:95
zyppng::ProvideFileSpec::ProvideFileSpec
ProvideFileSpec()
Definition providespec.cc:192
zyppng::ProvideMediaHandle
Definition provide.h:27
zyppng::ProvideRes
Definition provide.h:44
zyppng::expected
Definition expected.h:56
zyppng::expected::success
static expected success(ConsParams &&...params)
Definition expected.h:178
zyppng::expected::error
E & error()
Definition expected.h:288
zypp::str::regex_match
bool regex_match(const char *s, smatch &matches, const regex &regex) ZYPP_API
Regular expression matching.
Definition Regex.cc:80
std
Definition ansi.h:855
zypp::media
Definition MediaNetworkRequestExecutor.h:22
zypp::sat::empty
bool empty() const
Whether neither idents nor provides are set.
zypp::str::hasPrefix
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition String.h:1097
zypp::str::endsWith
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition String.h:1162
zypp::str::split
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t", const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition String.h:602
zyppng::DownloadWorkflow::provideToCacheDir
MaybeAwaitable< expected< zypp::ManagedFile > > provideToCacheDir(CacheProviderContextRef cacheContext, ProvideMediaHandle medium, zypp::Pathname file, ProvideFileSpec filespec)
Definition downloadwf.cc:192
zyppng::RepoDownloaderWorkflow::downloadMediaInfo
auto downloadMediaInfo(MediaHandle &&mediaHandle, const zypp::filesystem::Pathname &destdir)
Definition repodownloaderwf.h:41
zyppng::RepoDownloaderWorkflow::downloadMasterIndex
MaybeAwaitable< expected< repo::DownloadContextRef > > downloadMasterIndex(repo::DownloadContextRef dl, ProvideMediaHandle mediaHandle, zypp::filesystem::Pathname masterIndex_r)
Definition repodownloaderwf.cc:422
zyppng::SuseTagsWorkflows
Download workflow namespace for SUSETags (YaST2) repositories Encapsulates all the knowledge of which...
Definition susetags.cc:28
zyppng::SuseTagsWorkflows::download
MaybeAwaitable< expected< repo::DownloadContextRef > > download(repo::DownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition susetags.cc:312
zyppng::SuseTagsWorkflows::repoStatus
MaybeAwaitable< expected< zypp::RepoStatus > > repoStatus(repo::DownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition susetags.cc:78
zyppng::operators
Definition expected.h:699
zyppng::operators::transform_collect
auto transform_collect(Transformation &&f)
Definition expected.h:778
zyppng::operators::and_then
auto and_then(Fun &&function)
Definition expected.h:708
zyppng::operators::finishProgress
auto finishProgress(ProgressObserverRef progressObserver, ProgressObserver::FinishResult result=ProgressObserver::Success)
Definition progressobserver.h:252
zyppng::operators::incProgress
auto incProgress(ProgressObserverRef progressObserver, double progrIncrease=1.0, std::optional< std::string > newStr={})
Definition progressobserver.h:229
zyppng::operators::inspect
auto inspect(Fun &&function)
Definition expected.h:722
repodownloaderwf.h
susetags.h