libzypp  17.36.7
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <utility>
22 #include <zypp-core/parser/Sysconfig>
23 #include <zypp/base/Gettext.h>
24 
25 #include <zypp/media/MediaCurl.h>
28 #include <zypp-curl/ProxyInfo>
29 #include <zypp-curl/auth/CurlAuthData>
30 #include <zypp-media/auth/CredentialManager>
31 #include <zypp-curl/CurlConfig>
33 #include <zypp/Target.h>
34 #include <zypp/ZYppFactory.h>
35 #include <zypp/ZConfig.h>
36 #include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
37 
38 #include <cstdlib>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/mount.h>
42 #include <dirent.h>
43 #include <unistd.h>
44 #include <glib.h>
45 
47 
48 using std::endl;
49 
50 namespace internal {
51  using namespace zypp;
52  struct ProgressData
53  {
54  ProgressData( CURL *curl, time_t timeout = 0, zypp::Url url = zypp::Url(),
55  zypp::ByteCount expectedFileSize_r = 0,
57 
58  void updateStats( curl_off_t dltotal = 0.0, curl_off_t dlnow = 0.0 );
59 
60  int reportProgress() const;
61 
62  CURL * curl()
63  { return _curl; }
64 
65  bool timeoutReached() const
66  { return _timeoutReached; }
67 
68  bool fileSizeExceeded() const
69  { return _fileSizeExceeded; }
70 
72  { return _expectedFileSize; }
73 
74  void expectedFileSize( ByteCount newval_r )
75  { _expectedFileSize = newval_r; }
76 
77  private:
78  CURL * _curl;
80  time_t _timeout;
85 
86  time_t _timeStart = 0;
87  time_t _timeLast = 0;
88  time_t _timeRcv = 0;
89  time_t _timeNow = 0;
90 
91  curl_off_t _dnlTotal = 0.0;
92  curl_off_t _dnlLast = 0.0;
93  curl_off_t _dnlNow = 0.0;
94 
95  int _dnlPercent= 0;
96 
97  double _drateTotal= 0.0;
98  double _drateLast = 0.0;
99  };
100 
101 
102 
104  : _curl( curl )
105  , _url(std::move( url ))
106  , _timeout( timeout )
107  , _timeoutReached( false )
108  , _fileSizeExceeded ( false )
109  , _expectedFileSize( expectedFileSize_r )
110  , report( _report )
111  {}
112 
113  void ProgressData::updateStats( curl_off_t dltotal, curl_off_t dlnow )
114  {
115  time_t now = _timeNow = time(0);
116 
117  // If called without args (0.0), recompute based on the last values seen
118  if ( dltotal && dltotal != _dnlTotal )
119  _dnlTotal = dltotal;
120 
121  if ( dlnow && dlnow != _dnlNow )
122  {
123  _timeRcv = now;
124  _dnlNow = dlnow;
125  }
126 
127  // init or reset if time jumps back
128  if ( !_timeStart || _timeStart > now )
129  _timeStart = _timeLast = _timeRcv = now;
130 
131  // timeout condition
132  if ( _timeout )
133  _timeoutReached = ( (now - _timeRcv) > _timeout );
134 
135  // check if the downloaded data is already bigger than what we expected
136  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
137 
138  // percentage:
139  if ( _dnlTotal )
140  _dnlPercent = int( _dnlNow * 100 / _dnlTotal );
141 
142  // download rates:
143  _drateTotal = double(_dnlNow) / std::max( int(now - _timeStart), 1 );
144 
145  if ( _timeLast < now )
146  {
147  _drateLast = double(_dnlNow - _dnlLast) / int(now - _timeLast);
148  // start new period
149  _timeLast = now;
150  _dnlLast = _dnlNow;
151  }
152  else if ( _timeStart == _timeLast )
154  }
155 
157  {
158  if ( _fileSizeExceeded )
159  return 1;
160  if ( _timeoutReached )
161  return 1; // no-data timeout
162  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
163  return 1; // user requested abort
164  return 0;
165  }
166 
171  {
172  public:
174  const std::string & err_r,
175  const std::string & msg_r )
176  : media::MediaCurlException( url_r, err_r, msg_r )
177  {}
178  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
179  };
180 
181 }
182 
183 
184 using namespace internal;
185 using namespace zypp::base;
186 
187 namespace zypp {
188 
189  namespace media {
190 
191 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
192 
193 // we use this define to unbloat code as this C setting option
194 // and catching exception is done frequently.
196 #define SET_OPTION(opt,val) do { \
197  ret = curl_easy_setopt ( _curl, opt, val ); \
198  if ( ret != 0) { \
199  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
200  } \
201  } while ( false )
202 
203 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
204 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
205 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
206 
207 MediaCurl::MediaCurl( const Url & url_r,
208  const Pathname & attach_point_hint_r )
209  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
210  "/", // urlpath at attachpoint
211  true ), // does_download
212  _curl( NULL ),
213  _customHeaders(0L)
214 {
215  _curlError[0] = '\0';
216 
217  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
218 
220 
221  if( !attachPoint().empty())
222  {
223  PathInfo ainfo(attachPoint());
224  Pathname apath(attachPoint() + "XXXXXX");
225  char *atemp = ::strdup( apath.asString().c_str());
226  char *atest = NULL;
227  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
228  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
229  {
230  WAR << "attach point " << ainfo.path()
231  << " is not useable for " << url_r.getScheme() << endl;
232  setAttachPoint("", true);
233  }
234  else if( atest != NULL)
235  ::rmdir(atest);
236 
237  if( atemp != NULL)
238  ::free(atemp);
239  }
240 }
241 
243 {
245 }
246 
247 void MediaCurl::setCookieFile( const Pathname &fileName )
248 {
249  _cookieFile = fileName;
250 }
251 
252 void MediaCurl::setCurlError(const char* error)
253 {
254  // FIXME(dmllr): Use strlcpy if available for better performance
255  strncpy(_curlError, error, sizeof(_curlError)-1);
256  _curlError[sizeof(_curlError)-1] = '\0';
257 }
258 
260 
261 void MediaCurl::checkProtocol(const Url &url) const
262 {
263  curl_version_info_data *curl_info = NULL;
264  curl_info = curl_version_info(CURLVERSION_NOW);
265  // curl_info does not need any free (is static)
266  if (curl_info->protocols)
267  {
268  const char * const *proto = nullptr;
269  std::string scheme( url.getScheme());
270  bool found = false;
271  for(proto=curl_info->protocols; !found && *proto; ++proto)
272  {
273  if( scheme == std::string((const char *)*proto))
274  found = true;
275  }
276  if( !found)
277  {
278  std::string msg("Unsupported protocol '");
279  msg += scheme;
280  msg += "'";
282  }
283  }
284 }
285 
287 {
289 
290  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
291  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
292  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
293  if ( ret != 0 ) {
294  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
295  }
296 
297  SET_OPTION(CURLOPT_FAILONERROR, 1L);
298  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
299 
300  // create non persistant settings
301  // so that we don't add headers twice
302  TransferSettings vol_settings(_settings);
303 
304  // add custom headers for download.opensuse.org (bsc#955801)
305  if ( _url.getHost() == "download.opensuse.org" )
306  {
307  vol_settings.addHeader(anonymousIdHeader());
308  vol_settings.addHeader(distributionFlavorHeader());
309  }
310  vol_settings.addHeader("Pragma:");
311 
313 
314  // fill some settings from url query parameters
315  try
316  {
318  }
319  catch ( const MediaException &e )
320  {
321  disconnectFrom();
322  ZYPP_RETHROW(e);
323  }
324  // if the proxy was not set (or explicitly unset) by url, then look...
325  if ( _settings.proxy().empty() )
326  {
327  // ...at the system proxy settings
329  }
330 
332  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
333  {
334  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
335  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
336  }
337 
341  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
342  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
343  // just in case curl does not trigger its progress callback frequently
344  // enough.
345  if ( _settings.timeout() )
346  {
347  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
348  }
349 
350  // follow any Location: header that the server sends as part of
351  // an HTTP header (#113275)
352  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
353  // 3 redirects seem to be too few in some cases (bnc #465532)
354  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
355 
356  if ( _url.getScheme() == "https" )
357  {
358  if ( :: internal::setCurlRedirProtocols ( _curl ) != CURLE_OK ) {
360  }
361 
364  {
366  }
367 
369  {
370  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
371  }
372  if( ! _settings.clientKeyPath().empty() )
373  {
374  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
375  }
376 
377 #ifdef CURLSSLOPT_ALLOW_BEAST
378  // see bnc#779177
379  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
380  if ( ret != 0 ) {
381  disconnectFrom();
383  }
384 #endif
385  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
386  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
387  // bnc#903405 - POODLE: libzypp should only talk TLS
388  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
389  }
390 
391  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
392 
393  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
394  * We should proactively add the password to the request if basic auth is configured
395  * and a password is available in the credentials but not in the URL.
396  *
397  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
398  * and ask the server first about the auth method
399  */
400  if ( _settings.authType() == "basic"
401  && _settings.username().size()
402  && !_settings.password().size() ) {
403 
404  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
405  const auto cred = cm.getCred( _url );
406  if ( cred && cred->valid() ) {
407  if ( !_settings.username().size() )
408  _settings.setUsername(cred->username());
409  _settings.setPassword(cred->password());
410  }
411  }
412 
413  /*---------------------------------------------------------------*
414  CURLOPT_USERPWD: [user name]:[password]
415 
416  Url::username/password -> CURLOPT_USERPWD
417  If not provided, anonymous FTP identification
418  *---------------------------------------------------------------*/
419 
420  if ( _settings.userPassword().size() )
421  {
422  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
423  std::string use_auth = _settings.authType();
424  if (use_auth.empty())
425  use_auth = "digest,basic"; // our default
426  long auth = CurlAuthData::auth_type_str2long(use_auth);
427  if( auth != CURLAUTH_NONE)
428  {
429  DBG << "Enabling HTTP authentication methods: " << use_auth
430  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
431  SET_OPTION(CURLOPT_HTTPAUTH, auth);
432  }
433  }
434 
435  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
436  {
437  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
438  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
439  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
440  /*---------------------------------------------------------------*
441  * CURLOPT_PROXYUSERPWD: [user name]:[password]
442  *
443  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
444  * If not provided, $HOME/.curlrc is evaluated
445  *---------------------------------------------------------------*/
446 
447  std::string proxyuserpwd = _settings.proxyUserPassword();
448 
449  if ( proxyuserpwd.empty() )
450  {
451  CurlConfig curlconf;
452  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
453  if ( curlconf.proxyuserpwd.empty() )
454  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
455  else
456  {
457  proxyuserpwd = curlconf.proxyuserpwd;
458  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
459  }
460  }
461  else
462  {
463  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
464  }
465 
466  if ( ! proxyuserpwd.empty() )
467  {
468  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
469  }
470  }
471 #if CURLVERSION_AT_LEAST(7,19,4)
472  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
473  {
474  // Explicitly disabled in URL (see fillSettingsFromUrl()).
475  // This should also prevent libcurl from looking into the environment.
476  DBG << "Proxy: explicitly NOPROXY" << endl;
477  SET_OPTION(CURLOPT_NOPROXY, "*");
478  }
479 #endif
480  else
481  {
482  DBG << "Proxy: not explicitly set" << endl;
483  DBG << "Proxy: libcurl may look into the environment" << endl;
484  }
485 
487  if ( _settings.minDownloadSpeed() != 0 )
488  {
489  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
490  // default to 10 seconds at low speed
491  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
492  }
493 
494 #if CURLVERSION_AT_LEAST(7,15,5)
495  if ( _settings.maxDownloadSpeed() != 0 )
496  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
497 #endif
498 
499  /*---------------------------------------------------------------*
500  *---------------------------------------------------------------*/
501 
503  if ( ::geteuid() == 0 || PathInfo(_currentCookieFile).owner() == ::geteuid() )
505 
506  const auto &cookieFileParam = _url.getQueryParam( "cookies" );
507  if ( !cookieFileParam.empty() && str::strToBool( cookieFileParam, true ) )
508  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
509  else
510  MIL << "No cookies requested" << endl;
511  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
512  SET_OPTION(CURLOPT_XFERINFOFUNCTION, &progressCallback );
513  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
514 
515 #if CURLVERSION_AT_LEAST(7,18,0)
516  // bnc #306272
517  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
518 #endif
519  // Append settings custom headers to curl.
520  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
521  for ( const auto &header : vol_settings.headers() ) {
522  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
523  if ( !_customHeaders )
525  }
526  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
527 }
528 
530 
531 
532 void MediaCurl::attachTo (bool next)
533 {
534  if ( next )
536 
537  if ( !_url.isValid() )
539 
542  {
544  }
545 
546  disconnectFrom(); // clean _curl if needed
547  _curl = curl_easy_init();
548  if ( !_curl ) {
550  }
551  try
552  {
553  setupEasy();
554  }
555  catch (Exception & ex)
556  {
557  disconnectFrom();
558  ZYPP_RETHROW(ex);
559  }
560 
561  // FIXME: need a derived class to propelly compare url's
563  setMediaSource(media);
564 }
565 
566 bool
568 {
569  return MediaHandler::checkAttachPoint( apoint, true, true);
570 }
571 
573 
575 {
576  if ( _customHeaders )
577  {
578  curl_slist_free_all(_customHeaders);
579  _customHeaders = 0L;
580  }
581 
582  if ( _curl )
583  {
584  // bsc#1201092: Strange but within global_dtors we may exceptions here.
585  try { curl_easy_cleanup( _curl ); }
586  catch (...) { ; }
587  _curl = NULL;
588  }
589 }
590 
592 
593 void MediaCurl::releaseFrom( const std::string & ejectDev )
594 {
595  disconnect();
596 }
597 
599 
600 void MediaCurl::getFile( const OnMediaLocation &file ) const
601 {
602  // Use absolute file name to prevent access of files outside of the
603  // hierarchy below the attach point.
604  getFileCopy( file, localPath(file.filename()).absolutename() );
605 }
606 
608 
609 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
610 {
611 
612  const auto &filename = srcFile.filename();
613 
614  // Optional files will send no report until data are actually received (we know it exists).
615  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
617 
618  Url fileurl(getFileUrl(filename));
619 
620  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
621  unsigned internalTry = 0;
622  static constexpr unsigned maxInternalTry = 3;
623 
624  do
625  {
626  try
627  {
628  doGetFileCopy( srcFile, target, report );
629  break; // success!
630  }
631  // retry with proper authentication data
632  catch (MediaUnauthorizedException & ex_r)
633  {
634  if ( authenticate(ex_r.hint(), firstAuth) ) {
635  firstAuth = false; // must not return stored credentials again
636  continue; // retry
637  }
638 
639  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
640  ZYPP_RETHROW(ex_r);
641  }
642  // unexpected exception
643  catch (MediaException & excpt_r)
644  {
645  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
646  ++internalTry;
647  if ( internalTry < maxInternalTry ) {
648  // just report (NO_ERROR); no interactive request to the user
649  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
650  continue; // retry
651  }
652  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
653  }
654 
656  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
657  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
658  {
660  }
661  report->finish(fileurl, reason, excpt_r.asUserHistory());
662  ZYPP_RETHROW(excpt_r);
663  }
664  }
665  while ( true );
666  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
667 }
668 
670 
671 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
672 {
673  bool retry = false;
674 
675  do
676  {
677  try
678  {
679  return doGetDoesFileExist( filename );
680  }
681  // authentication problem, retry with proper authentication data
682  catch (MediaUnauthorizedException & ex_r)
683  {
684  if(authenticate(ex_r.hint(), !retry))
685  retry = true;
686  else
687  ZYPP_RETHROW(ex_r);
688  }
689  // unexpected exception
690  catch (MediaException & excpt_r)
691  {
692  ZYPP_RETHROW(excpt_r);
693  }
694  }
695  while (retry);
696 
697  return false;
698 }
699 
701 
703  CURLcode code,
704  bool timeout_reached) const
705 {
706  if ( code != 0 )
707  {
708  Url url;
709  if (filename.empty())
710  url = _url;
711  else
712  url = getFileUrl(filename);
713 
714  std::string err;
715  {
716  switch ( code )
717  {
718  case CURLE_UNSUPPORTED_PROTOCOL:
719  err = " Unsupported protocol";
720  if ( !_lastRedirect.empty() )
721  {
722  err += " or redirect (";
723  err += _lastRedirect;
724  err += ")";
725  }
726  break;
727  case CURLE_URL_MALFORMAT:
728  case CURLE_URL_MALFORMAT_USER:
729  err = " Bad URL";
730  break;
731  case CURLE_LOGIN_DENIED:
732  ZYPP_THROW(
733  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
734  break;
735  case CURLE_HTTP_RETURNED_ERROR:
736  {
737  long httpReturnCode = 0;
738  CURLcode infoRet = curl_easy_getinfo( _curl,
739  CURLINFO_RESPONSE_CODE,
740  &httpReturnCode );
741  if ( infoRet == CURLE_OK )
742  {
743  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
744  switch ( httpReturnCode )
745  {
746  case 401:
747  {
748  std::string auth_hint = getAuthHint();
749 
750  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
751  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
752 
754  url, "Login failed.", _curlError, auth_hint
755  ));
756  }
757 
758  case 502: // bad gateway (bnc #1070851)
759  case 503: // service temporarily unavailable (bnc #462545)
761  case 504: // gateway timeout
763  case 403:
764  {
765  std::string msg403;
766  if ( url.getHost().find(".suse.com") != std::string::npos )
767  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
768  else if (url.asString().find("novell.com") != std::string::npos)
769  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
771  }
772  case 404:
773  case 410:
775  }
776 
777  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
779  }
780  else
781  {
782  std::string msg = "Unable to retrieve HTTP response:";
783  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
785  }
786  }
787  break;
788  case CURLE_FTP_COULDNT_RETR_FILE:
789 #if CURLVERSION_AT_LEAST(7,16,0)
790  case CURLE_REMOTE_FILE_NOT_FOUND:
791 #endif
792  case CURLE_FTP_ACCESS_DENIED:
793  case CURLE_TFTP_NOTFOUND:
794  err = "File not found";
796  break;
797  case CURLE_BAD_PASSWORD_ENTERED:
798  case CURLE_FTP_USER_PASSWORD_INCORRECT:
799  err = "Login failed";
800  break;
801  case CURLE_COULDNT_RESOLVE_PROXY:
802  case CURLE_COULDNT_RESOLVE_HOST:
803  case CURLE_COULDNT_CONNECT:
804  case CURLE_FTP_CANT_GET_HOST:
805  err = "Connection failed";
806  break;
807  case CURLE_WRITE_ERROR:
808  err = "Write error";
809  break;
810  case CURLE_PARTIAL_FILE:
811  case CURLE_OPERATION_TIMEDOUT:
812  timeout_reached = true; // fall though to TimeoutException
813  // fall though...
814  case CURLE_ABORTED_BY_CALLBACK:
815  if( timeout_reached )
816  {
817  err = "Timeout reached";
819  }
820  else
821  {
822  err = "User abort";
823  }
824  break;
825 
826  default:
827  err = "Curl error " + str::numstring( code );
828  break;
829  }
830 
831  // uhm, no 0 code but unknown curl exception
833  }
834  }
835  else
836  {
837  // actually the code is 0, nothing happened
838  }
839 }
840 
842 
843 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
844 {
845  DBG << filename.asString() << endl;
846 
847  if(!_url.isValid())
849 
850  if(_url.getHost().empty())
852 
853  Url url(getFileUrl(filename));
854 
855  DBG << "URL: " << url.asString() << endl;
856  // Use URL without options and without username and passwd
857  // (some proxies dislike them in the URL).
858  // Curl seems to need the just scheme, hostname and a path;
859  // the rest was already passed as curl options (in attachTo).
860  Url curlUrl( clearQueryString(url) );
861 
862  //
863  // See also Bug #154197 and ftp url definition in RFC 1738:
864  // The url "ftp://user@host/foo/bar/file" contains a path,
865  // that is relative to the user's home.
866  // The url "ftp://user@host//foo/bar/file" (or also with
867  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
868  // contains an absolute path.
869  //
870  _lastRedirect.clear();
871  std::string urlBuffer( curlUrl.asString());
872  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
873  urlBuffer.c_str() );
874  if ( ret != 0 ) {
876  }
877 
878  // If no head requests allowed (?head_requests=no):
879  // Instead of returning no data with NOBODY, we return
880  // little data, that works with broken servers, and
881  // works for ftp as well, because retrieving only headers
882  // ftp will return always OK code ?
883  // See http://curl.haxx.se/docs/knownbugs.html #58
885  struct TempSetHeadRequest
886  {
887  TempSetHeadRequest(CURL *curl_r, bool doHttpHeadRequest_r)
888  : _curl{curl_r}, _doHttpHeadRequest{doHttpHeadRequest_r} {
889  if ( _doHttpHeadRequest ) {
890  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
891  } else {
892  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
893  }
894  }
895  TempSetHeadRequest(const TempSetHeadRequest &) = delete;
896  TempSetHeadRequest(TempSetHeadRequest &&) = delete;
897  TempSetHeadRequest &operator=(const TempSetHeadRequest &) = delete;
898  TempSetHeadRequest &operator=(TempSetHeadRequest &&) = delete;
899  ~TempSetHeadRequest() {
900  if ( _doHttpHeadRequest ) {
901  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
902  /* yes, this is why we never got to get NOBODY working before,
903  because setting it changes this option too, and we also*
904  need to reset it
905  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
906  */
907  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
908  } else {
909  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
910  }
911  }
912  private:
913  CURL * _curl;
914  bool _doHttpHeadRequest;
915  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
916 
917 
918  AutoFILE file { ::fopen( "/dev/null", "w" ) };
919  if ( !file ) {
920  ERR << "fopen failed for /dev/null" << endl;
921  ZYPP_THROW(MediaWriteException("/dev/null"));
922  }
923 
924  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
925  if ( ret != 0 ) {
927  }
928 
929  CURLcode ok = executeCurl();
930  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
931 
932  // as we are not having user interaction, the user can't cancel
933  // the file existence checking, a callback or timeout return code
934  // will be always a timeout.
935  try {
936  evaluateCurlCode( filename, ok, true /* timeout */);
937  }
938  catch ( const MediaFileNotFoundException &e ) {
939  // if the file did not exist then we can return false
940  return false;
941  }
942  catch ( const MediaException &e ) {
943  // some error, we are not sure about file existence, rethrw
944  ZYPP_RETHROW(e);
945  }
946  // exists
947  return ( ok == CURLE_OK );
948 }
949 
951 
952 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
953 {
954  Pathname dest = target.absolutename();
955  if( assert_dir( dest.dirname() ) )
956  {
957  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
958  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
959  }
960 
961  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
962  AutoFILE file;
963  {
964  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
965  if( ! buf )
966  {
967  ERR << "out of memory for temp file name" << endl;
968  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
969  }
970 
971  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
972  if( tmp_fd == -1 )
973  {
974  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
976  }
977  destNew = ManagedFile( (*buf), filesystem::unlink );
978 
979  file = ::fdopen( tmp_fd, "we" );
980  if ( ! file )
981  {
982  ERR << "fopen failed for file '" << destNew << "'" << endl;
984  }
985  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
986  }
987 
988  DBG << "dest: " << dest << endl;
989  DBG << "temp: " << destNew << endl;
990 
991  // set IFMODSINCE time condition (no download if not modified)
992  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
993  {
994  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
995  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
996  }
997  else
998  {
999  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1000  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1001  }
1002  try
1003  {
1004  doGetFileCopyFile( srcFile, dest, file, report, options);
1005  }
1006  catch (Exception &e)
1007  {
1008  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1009  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1010  ZYPP_RETHROW(e);
1011  }
1012 
1013  long httpReturnCode = 0;
1014  CURLcode infoRet = curl_easy_getinfo(_curl,
1015  CURLINFO_RESPONSE_CODE,
1016  &httpReturnCode);
1017  bool modified = true;
1018  if (infoRet == CURLE_OK)
1019  {
1020  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1021  if ( httpReturnCode == 304
1022  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1023  {
1024  DBG << " Not modified.";
1025  modified = false;
1026  }
1027  DBG << endl;
1028  }
1029  else
1030  {
1031  WAR << "Could not get the response code." << endl;
1032  }
1033 
1034  if (modified || infoRet != CURLE_OK)
1035  {
1036  // apply umask
1037  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1038  {
1039  ERR << "Failed to chmod file " << destNew << endl;
1040  }
1041 
1042  file.resetDispose(); // we're going to close it manually here
1043  if ( ::fclose( file ) )
1044  {
1045  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1046  ZYPP_THROW(MediaWriteException(destNew));
1047  }
1048 
1049  // move the temp file into dest
1050  if ( rename( destNew, dest ) != 0 ) {
1051  ERR << "Rename failed" << endl;
1053  }
1054  destNew.resetDispose(); // no more need to unlink it
1055  }
1056 
1057  DBG << "done: " << PathInfo(dest) << endl;
1058 }
1059 
1061 
1062 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1063 {
1064  DBG << srcFile.filename().asString() << endl;
1065 
1066  if(!_url.isValid())
1068 
1069  if(_url.getHost().empty())
1071 
1072  Url url(getFileUrl(srcFile.filename()));
1073 
1074  DBG << "URL: " << url.asString() << endl;
1075  // Use URL without options and without username and passwd
1076  // (some proxies dislike them in the URL).
1077  // Curl seems to need the just scheme, hostname and a path;
1078  // the rest was already passed as curl options (in attachTo).
1079  Url curlUrl( clearQueryString(url) );
1080 
1081  //
1082  // See also Bug #154197 and ftp url definition in RFC 1738:
1083  // The url "ftp://user@host/foo/bar/file" contains a path,
1084  // that is relative to the user's home.
1085  // The url "ftp://user@host//foo/bar/file" (or also with
1086  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1087  // contains an absolute path.
1088  //
1089  _lastRedirect.clear();
1090  std::string urlBuffer( curlUrl.asString());
1091  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1092  urlBuffer.c_str() );
1093  if ( ret != 0 ) {
1095  }
1096 
1097  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1098  if ( ret != 0 ) {
1100  }
1101 
1102  // Set callback and perform.
1103  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1104  if (!(options & OPTION_NO_REPORT_START))
1105  report->start(url, dest);
1106  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1107  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1108  }
1109 
1110  ret = executeCurl();
1111 #if CURLVERSION_AT_LEAST(7,19,4)
1112  // bnc#692260: If the client sends a request with an If-Modified-Since header
1113  // with a future date for the server, the server may respond 200 sending a
1114  // zero size file.
1115  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1116  if ( ftell(file) == 0 && ret == 0 )
1117  {
1118  long httpReturnCode = 33;
1119  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1120  {
1121  long conditionUnmet = 33;
1122  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1123  {
1124  WAR << "TIMECONDITION unmet - retry without." << endl;
1125  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1126  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1127  ret = executeCurl();
1128  }
1129  }
1130  }
1131 #endif
1132 
1133  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1134  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1135  }
1136 
1137  if ( ret != 0 )
1138  {
1139  ERR << "curl error: " << ret << ": " << _curlError
1140  << ", temp file size " << ftell(file)
1141  << " bytes." << endl;
1142 
1143  // the timeout is determined by the progress data object
1144  // which holds whether the timeout was reached or not,
1145  // otherwise it would be a user cancel
1146  try {
1147 
1148  if ( progressData.fileSizeExceeded() )
1149  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1150 
1151  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1152  }
1153  catch ( const MediaException &e ) {
1154  // some error, we are not sure about file existence, rethrw
1155  ZYPP_RETHROW(e);
1156  }
1157  }
1158 }
1159 
1161 
1162 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1163 {
1164  filesystem::DirContent content;
1165  getDirInfo( content, dirname, /*dots*/false );
1166 
1167  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1168  Pathname filename = dirname + it->name;
1169  int res = 0;
1170 
1171  switch ( it->type ) {
1172  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1173  case filesystem::FT_FILE:
1174  getFile( OnMediaLocation( filename ) );
1175  break;
1176  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1177  if ( recurse_r ) {
1178  getDir( filename, recurse_r );
1179  } else {
1180  res = assert_dir( localPath( filename ) );
1181  if ( res ) {
1182  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1183  }
1184  }
1185  break;
1186  default:
1187  // don't provide devices, sockets, etc.
1188  break;
1189  }
1190  }
1191 }
1192 
1194 
1195 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1196  const Pathname & dirname, bool dots ) const
1197 {
1198  getDirectoryYast( retlist, dirname, dots );
1199 }
1200 
1202 
1204  const Pathname & dirname, bool dots ) const
1205 {
1206  getDirectoryYast( retlist, dirname, dots );
1207 }
1208 
1210 //
1211 int MediaCurl::aliveCallback( void *clientp, curl_off_t /*dltotal*/, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/ )
1212 {
1213  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1214  if( pdata )
1215  {
1216  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1217  // prevent a percentage raise while downloading a metalink file. Download
1218  // activity however is indicated by propagating the download rate (via dlnow).
1219  pdata->updateStats( 0.0, dlnow );
1220  return pdata->reportProgress();
1221  }
1222  return 0;
1223 }
1224 
1225 int MediaCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
1226 {
1227  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1228  if( pdata )
1229  {
1230  // work around curl bug that gives us old data
1231  long httpReturnCode = 0;
1232  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1233  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1234 
1235  pdata->updateStats( dltotal, dlnow );
1236  return pdata->reportProgress();
1237  }
1238  return 0;
1239 }
1240 
1242 {
1243  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1244  return pdata ? pdata->curl() : 0;
1245 }
1246 
1248 
1249 std::string MediaCurl::getAuthHint() const
1250 {
1251  long auth_info = CURLAUTH_NONE;
1252 
1253  CURLcode infoRet =
1254  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1255 
1256  if(infoRet == CURLE_OK)
1257  {
1258  return CurlAuthData::auth_type_long2str(auth_info);
1259  }
1260 
1261  return "";
1262 }
1263 
1268 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1269 {
1270  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1271  if ( data ) {
1272  data->expectedFileSize( expectedFileSize );
1273  }
1274 }
1275 
1281 CURLcode MediaCurl::executeCurl() const
1282 {
1283  // initialize our helpers
1285  internal::CurlPollHelper::CurlPoll{ curl_multi_init() }
1286  ,[](auto &releaseMe ){ if (releaseMe._multi) curl_multi_cleanup(releaseMe._multi); }
1287  );
1288 
1289  if (!cMulti->_multi)
1291 
1292  // we could derive from that, but currently that does not make a lot of sense
1293  internal::CurlPollHelper _curlHelper(cMulti.value());
1294 
1295  // add the easy handle to the multi instance
1296  if ( curl_multi_add_handle( cMulti->_multi, _curl ) != CURLM_OK )
1297  ZYPP_THROW(MediaCurlException( _url, "curl_multi_add_handle", "unknown error"));
1298 
1299  // make sure the handle is cleanly removed from the multi handle
1300  OnScopeExit autoRemove([&](){ curl_multi_remove_handle( cMulti->_multi, _curl ); });
1301 
1302  // kickstart curl, this will cause libcurl to go over the added handles and register sockets and timeouts
1303  CURLMcode mcode = _curlHelper.handleTimout();
1304  if (mcode != CURLM_OK)
1305  ZYPP_THROW(MediaCurlException( _url, "curl_multi_socket_action", "unknown error"));
1306 
1307  bool canContinue = true;
1308  while ( canContinue ) {
1309 
1310  CURLMsg *msg = nullptr;
1311  int nqueue = 0;
1312  while ((msg = curl_multi_info_read( cMulti->_multi, &nqueue)) != 0) {
1313  if ( msg->msg != CURLMSG_DONE ) continue;
1314  if ( msg->easy_handle != _curl ) continue;
1315 
1316  return msg->data.result;
1317  }
1318 
1319  // copy watched sockets in case curl changes the vector as we go over the events later
1320  std::vector<GPollFD> requestedFds = _curlHelper.socks;
1321 
1322  int r = zypp_detail::zypp_poll( requestedFds, _curlHelper.timeout_ms.value_or( -1 ) );
1323  if ( r == -1 )
1324  ZYPP_THROW( MediaCurlException(_url, "zypp_poll() failed", "unknown error") );
1325 
1326  // run curl
1327  if ( r == 0 ) {
1328  CURLMcode mcode = _curlHelper.handleTimout();
1329  if (mcode != CURLM_OK)
1330  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1331  } else {
1332  CURLMcode mcode = _curlHelper.handleSocketActions( requestedFds );
1333  if (mcode != CURLM_OK)
1334  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1335  }
1336  }
1337  return CURLE_OK;
1338 }
1339 
1341 
1342 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1343 {
1345  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1346  CurlAuthData_Ptr credentials;
1347 
1348  // get stored credentials
1349  AuthData_Ptr cmcred = cm.getCred(_url);
1350 
1351  if (cmcred && firstTry)
1352  {
1353  credentials.reset(new CurlAuthData(*cmcred));
1354  DBG << "got stored credentials:" << endl << *credentials << endl;
1355  }
1356  // if not found, ask user
1357  else
1358  {
1359 
1360  CurlAuthData_Ptr curlcred;
1361  curlcred.reset(new CurlAuthData());
1363 
1364  // preset the username if present in current url
1365  if (!_url.getUsername().empty() && firstTry)
1366  curlcred->setUsername(_url.getUsername());
1367  // if CM has found some credentials, preset the username from there
1368  else if (cmcred)
1369  curlcred->setUsername(cmcred->username());
1370 
1371  // indicate we have no good credentials from CM
1372  cmcred.reset();
1373 
1374  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1375 
1376  // set available authentication types from the exception
1377  // might be needed in prompt
1378  curlcred->setAuthType(availAuthTypes);
1379 
1380  // ask user
1381  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1382  {
1383  DBG << "callback answer: retry" << endl
1384  << "CurlAuthData: " << *curlcred << endl;
1385 
1386  if (curlcred->valid())
1387  {
1388  credentials = curlcred;
1389  // if (credentials->username() != _url.getUsername())
1390  // _url.setUsername(credentials->username());
1398  }
1399  }
1400  else
1401  {
1402  DBG << "callback answer: cancel" << endl;
1403  }
1404  }
1405 
1406  // set username and password
1407  if (credentials)
1408  {
1409  // HACK, why is this const?
1410  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1411  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1412 
1413  // set username and password
1414  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1416 
1417  // set available authentication types from the exception
1418  if (credentials->authType() == CURLAUTH_NONE)
1419  credentials->setAuthType(availAuthTypes);
1420 
1421  // set auth type (seems this must be set _after_ setting the userpwd)
1422  if (credentials->authType() != CURLAUTH_NONE)
1423  {
1424  // FIXME: only overwrite if not empty?
1425  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1426  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1428  }
1429 
1430  if (!cmcred)
1431  {
1432  credentials->setUrl(_url);
1433  cm.addCred(*credentials);
1434  cm.save();
1435  }
1436 
1437  return true;
1438  }
1439 
1440  return false;
1441 }
1442 
1443 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1445 
1446  } // namespace media
1447 } // namespace zypp
1448 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:551
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:567
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:173
Interface to gettext.
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:203
#define MIL
Definition: Logger.h:100
std::string curlUnEscape(const std::string &text_r)
Definition: curlhelper.cc:373
const Pathname & clientCertificatePath() const
SSL client certificate file.
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
#define _(MSG)
Definition: Gettext.h:39
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:143
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:251
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:424
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:940
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:261
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1342
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1210
void setPassword(const std::string &val_r)
sets the auth password
ByteCount _expectedFileSize
Definition: MediaCurl.cc:83
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Store and operate with byte count.
Definition: ByteCount.h:31
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:175
int reportProgress() const
Definition: MediaCurl.cc:156
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:380
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
const char * c_str() const
String representation.
Definition: Pathname.h:112
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:176
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:609
MediaCurlException(const Url &url_r, std::string err_r, std::string msg_r)
time_t _timeNow
Now.
Definition: MediaCurl.cc:89
Definition: Arch.h:363
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:286
bool timeoutReached() const
Definition: MediaCurl.cc:65
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
CURLcode executeCurl() const
Definition: MediaCurl.cc:1281
AutoDispose<int> calling ::close
Definition: AutoDispose.h:309
std::string _currentCookieFile
Definition: MediaCurl.h:163
const std::string & password() const
auth password
void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1162
void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:600
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:526
#define ERR
Definition: Logger.h:102
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:68
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers (trimmed)
zypp::Url _url
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:247
void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:593
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1268
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:116
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:444
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:515
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:678
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:242
void setAuthType(const std::string &val_r)
set the allowed authentication types
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:93
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:702
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:127
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:101
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:81
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:183
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1225
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:507
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:843
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:74
const Pathname & filename() const
The path to the resource on the medium.
void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:532
std::string numstring(char n, int w=0)
Definition: String.h:289
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:705
bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:671
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:171
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1062
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:95
curl_slist * _customHeaders
Definition: MediaCurl.h:171
bool proxyEnabled() const
proxy is enabled
constexpr std::string_view FILE("file")
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:196
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:141
Base class for Exception.
Definition: Exception.h:146
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:167
curl_off_t _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:93
Url url() const
Url used.
Definition: MediaHandler.h:503
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
curl_off_t _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:91
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:98
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:606
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:86
void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1195
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:130
void disconnectFrom() override
Definition: MediaCurl.cc:574
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1241
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
void setCurlError(const char *error)
Definition: MediaCurl.cc:252
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
curl_off_t _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:92
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:320
static Pathname _cookieFile
Definition: MediaCurl.h:164
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:338
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:97
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:806
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1249
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:88
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:23
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:747
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Definition: ZYppImpl.cc:313
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:952
void updateStats(curl_off_t dltotal=0.0, curl_off_t dlnow=0.0)
Definition: MediaCurl.cc:113
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: Application.cc:19
Curl HTTP authentication data.
Definition: curlauthdata.h:22
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:165
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:71
CURLcode setCurlRedirProtocols(CURL *curl)
Definition: curlhelper.cc:523
const std::string & proxy() const
proxy host
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:371
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:87
TrueBool _guard
Definition: TargetImpl.cc:1625
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1211
bool userMayRWX() const
Definition: PathInfo.h:361
const std::string & userAgentString() const
user agent string (trimmed)
Url manipulation class.
Definition: Url.h:92
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:99
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:170
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:84
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:590