45#if __cplusplus < 201103L
51 bool isNotSpace(
char c )
81 const std::string &name,
85 int value = defaultVal;
87 URL::ParamsMap::const_iterator it;
89 env->
GetInt( name, value );
90 it = url.
GetParams().find( std::string(
"XrdCl.") + name );
93 int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 );
104 const std::string &name,
105 const std::string &defaultVal )
108 std::string value = defaultVal;
109 URL::ParamsMap::const_iterator it;
112 it = url.
GetParams().find( std::string(
"XrdCl.") + name );
124 if( addressType ==
"IPv6" )
126 else if( addressType ==
"IPv4" )
128 else if( addressType ==
"IPv4Mapped6" )
130 else if( addressType ==
"IPAll" )
186 if( addresses.size() == 0 )
199 env->
GetInt(
"IPNoShuffle", ipNoShuffle );
203#if __cplusplus < 201103L
205 static struct only_once_t
209 std::srand (
unsigned ( std::time(0) ) );
213 std::random_shuffle( addresses.begin(), addresses.begin() + ordn );
214 std::random_shuffle( addresses.begin() + ordn, addresses.end() );
216 static std::default_random_engine rand_engine(
217 std::chrono::system_clock::now().time_since_epoch().count() );
219 std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine );
220 std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine );
235 const std::string &hostId,
236 std::vector<XrdNetAddr> &addresses )
239 std::vector<XrdNetAddr>::iterator it;
240 for( it = addresses.begin(); it != addresses.end(); ++it )
247 addrStr.erase( addrStr.length()-2, 2 );
248 log->
Debug( type,
"[%s] Found %d address(es): %s",
249 hostId.c_str(), addresses.size(), addrStr.c_str() );
259 time_t ttNow = timestamp;
260 localtime_r( &ttNow, &tsNow );
261 strftime( now, 30,
"%Y-%m-%d %H:%M:%S %z", &tsNow );
270 uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec;
271 uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec;
272 return endUSec-startUSec;
279 const std::string &checkSumType,
286 size_t pos = url.
GetPath().find(
'?' );
287 std::string cksPath = url.
GetPath() + ( pos == std::string::npos ?
'?' :
'&' ) +
"cks.type=" + checkSumType;
299 msg +=
" Got an error while querying the checksum!";
307 std::vector<std::string> elems;
311 if( elems.size() != 2 )
314 if( elems[0] != checkSumType )
317 checkSum = elems[0] +
":";
321 url.
GetPath().c_str(), checkSum.c_str() );
330 const std::string &checkSumType,
331 const std::string &path )
343 bool status = cksMan->
Calculate( ckSum, checkSumType, path.c_str() );
351 char *cksBuffer =
new char[265];
352 ckSum.
Get( cksBuffer, 256 );
353 checkSum = checkSumType +
":";
368 double final = bytes;
370 char suf[3] = {
'k',
'M',
'G' };
371 for( i = 0; i < 3 && final > 1024; ++i,
final /= 1024 ) {};
372 std::ostringstream o;
373 o << std::setprecision(4) <<
final;
374 if( i > 0 ) o << suf[i-1];
389 Buffer *queryResponse = 0;
396 server.c_str(), st.
ToStr().c_str() );
403 log->
Error(
UtilityMsg,
"Cannot query source data server: empty response." );
408 std::string answer = queryResponse->
ToString();
409 delete queryResponse;
410 if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0)
428 log->
Debug(
UtilityMsg,
"Checking if the data server %s supports tpc / tpc lite",
433 Buffer *queryResponse = 0;
440 server.c_str(), st.
ToStr().c_str() );
447 log->
Error(
UtilityMsg,
"Cannot query source data server: empty response." );
452 std::string answer = queryResponse->
ToString();
453 delete queryResponse;
457 log->
Error(
UtilityMsg,
"Cannot query source data server: empty response." );
462 std::vector<std::string> resp;
465 if( resp.empty() || resp[0].empty() ||
466 !isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 )
473 if( resp.size() == 1 || resp[1] ==
"tpcdlg" )
491 std::vector<std::string> el;
496 std::string cc = *el.rbegin();
497 if( cc.length() == 2 )
506 const std::string &path )
508 DIR *dp =
opendir( path.c_str() );
514 while( (dirEntry =
readdir(dp)) != 0 )
516 std::string entryName = dirEntry->d_name;
517 if( !entryName.compare( 0, 2,
"..") )
519 if( !entryName.compare( 0, 1,
".") )
522 entries.push_back( dirEntry->d_name );
534 const std::string &file )
537 std::ifstream inFile( file.c_str() );
543 while( std::getline( inFile, line ) )
545 if( line.empty() || line[0] ==
'#' )
548 std::vector<std::string> elems;
550 if( elems.size() != 2 )
552 std::string key = elems[0];
Trim( key );
553 std::string value = elems[1];
Trim( value );
566 const std::string &dir )
572 std::vector<std::string> entries;
577 dir.c_str(), st.
ToString().c_str() );
581 static const std::string suffix =
".conf";
582 for(
auto &entry : entries )
584 std::string confFile = dir +
"/" + entry;
586 if( confFile.length() <= suffix.length() )
588 if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) )
595 confFile.c_str(), st.
ToString().c_str() );
607 str.erase( str.begin(),
608 std::find_if( str.begin(), str.end(), isNotSpace ) );
609 str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(),
621 PropertyList::PropertyMap::const_iterator it;
623 for( it = list.
begin(); it != list.
end(); ++it )
624 keyVals +=
"'" + it->first +
"' = '" + it->second +
"', ";
625 keyVals.erase( keyVals.length()-2, 2 );
626 log->
Dump( topic, format, keyVals.c_str() );
634 char *hex =
new char[2*size+1];
635 for( uint16_t i = 0; i < size; ++i )
636 snprintf( hex+(2*i), 3,
"%02x", (
int)array[i] );
637 std::string result = hex;
646 const std::string &checksum )
648 if( name ==
"adler32" || name ==
"crc32" )
651 for( i = 0; i < checksum.length(); ++i )
652 if( checksum[i] !=
'0' )
654 return checksum.substr(i);
664 std::vector<std::string> ret;
672 std::string response = resp->
ToString();
673 if( response !=
"chksum" )
676 std::vector<std::string> result;
679 std::vector<std::string>::iterator itr = result.begin();
680 for( ; itr != result.end(); ++itr )
682 size_t pos = itr->find(
':' );
683 if( pos == std::string::npos )
continue;
684 std::string cksname = itr->substr( pos + 1 );
686 cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ),
688 ret.push_back( std::move( cksname ) );
707 bool open_new = ( options &
kXR_new );
708 if( open_wrt && !open_new )
return false;
712 URL::ParamsMap::const_iterator itr = params.find(
"xrdec.nbdta" );
713 if( itr == params.end() )
return false;
714 size_t nbdta = std::stoul( itr->second );
716 itr = params.find(
"xrdec.nbprt" );
717 if( itr == params.end() )
return false;
718 size_t nbprt = std::stoul( itr->second );
720 itr = params.find(
"xrdec.blksz" );
721 if( itr == params.end() )
return false;
723 itr = params.find(
"xrdec.plgr" );
724 if( itr == params.end() )
return false;
725 std::vector<std::string> plgr;
727 if( plgr.size() < nbdta + nbprt )
return false;
729 itr = params.find(
"xrdec.objid" );
730 if( itr == params.end() )
return false;
732 itr = params.find(
"xrdec.format" );
733 if( itr == params.end() )
return false;
734 size_t format = std::stoul( itr->second );
735 if( format != 1 )
return false;
737 itr = params.find(
"xrdec.dtacgi" );
738 if( itr != params.end() )
740 std::vector<std::string> dtacgi;
742 if( plgr.size() != dtacgi.size() )
return false;
745 itr = params.find(
"xrdec.mdtacgi" );
746 if( itr != params.end() )
748 std::vector<std::string> mdtacgi;
750 if( plgr.size() != mdtacgi.size() )
return false;
753 itr = params.find(
"xrdec.cosc" );
754 if( itr == params.end() )
return false;
755 std::string cosc = itr->second;
756 if( cosc !=
"true" && cosc !=
"false" )
return false;
778 std::set<std::string> local_supported;
779 local_supported.insert(
"adler32" );
780 local_supported.insert(
"crc32" );
781 local_supported.insert(
"md5" );
782 local_supported.insert(
"zcrc32" );
784 std::vector<std::string> srccks;
790 env->
GetInt(
"ZipMtlnCksum", useMtlnCksum );
796 if( !zip || ( zip && useMtlnCksum ) )
801 srccks.insert( srccks.end(), cks.begin(), cks.end() );
810 srccks.push_back(
"zcrc32" );
818 srccks.insert( srccks.end(), cks.begin(), cks.end() );
821 std::vector<std::string> dstcks;
830 dstcks.insert( dstcks.end(), cks.begin(), cks.end() );
841 std::vector<std::string>::iterator itr = dstcks.begin();
842 for( ; itr != dstcks.end(); ++itr )
843 if( local_supported.count( *itr ) )
return *itr;
844 return std::string();
852 std::vector<std::string>::iterator itr = srccks.begin();
853 for( ; itr != srccks.end(); ++itr )
854 if( local_supported.count( *itr ) )
return *itr;
855 return std::string();
862 std::set<std::string> dst_supported( dstcks.begin(), dstcks.end() );
863 std::vector<std::string>::iterator itr = srccks.begin();
864 for( ; itr != srccks.end(); ++itr )
865 if( dst_supported.count( *itr ) )
return *itr;
866 return std::string();
874 const uint32_t maxcs,
878 if( !chunks.size() )
return;
880 listsvec.emplace_back();
882 const size_t cs = chunks.size();
891 if( maxc && nc >= maxc )
893 listsvec.emplace_back();
894 c = &listsvec.back();
895 c->reserve( cs - idx );
902 if( maxcs && tmpc.
length > maxcs )
907 tmpc.
buffer =
static_cast<char*
>( tmpc.
buffer ) + maxcs;
struct ClientOpenRequest open
int Set(const char *csName)
int Get(char *Buff, int Blen)
Binary blob representation.
void FromString(const std::string str)
Fill the buffer from a string.
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
std::string ToString() const
Convert the buffer to a string.
Manage the checksum calc objects.
bool Calculate(XrdCksData &result, const std::string &algName, const std::string &filePath)
Calculate a checksum of for a given file.
static CheckSumManager * GetCheckSumManager()
Get checksum manager.
static Log * GetLog()
Get default log.
static Env * GetEnv()
Get default client environment.
bool GetString(const std::string &key, std::string &value)
bool GetInt(const std::string &key, int &value)
Send file/filesystem queries to an XRootD cluster.
XRootDStatus Query(QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
void Error(uint64_t topic, const char *format,...)
Report an error.
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
The message representation used throughout the system.
A key-value pair map storing both keys and values as strings.
PropertyMap::const_iterator end() const
Get the end iterator.
PropertyMap::const_iterator begin() const
Get the begin iterator.
Singleton access to URL to virtual redirector mapping.
static RedirectorRegistry & Instance()
Returns reference to the single instance.
VirtualRedirector * Get(const URL &url) const
Get a virtual redirector associated with the given URL.
const std::string & GetPath() const
Get the path.
bool IsMetalink() const
Is it a URL to a metalink.
const std::string & GetHostName() const
Get the name of the target host.
std::map< std::string, std::string > ParamsMap
const ParamsMap & GetParams() const
Get the URL params.
const std::string & GetProtocol() const
Get the protocol.
int GetPort() const
Get the target port.
static std::string TimeToString(time_t timestamp)
Convert timestamp to a string.
static XRootDStatus CheckTPCLite(const std::string &server, uint16_t timeout=0)
static void LogHostAddresses(Log *log, uint64_t type, const std::string &hostId, std::vector< XrdNetAddr > &addresses)
Log all the addresses on the list.
static std::string NormalizeChecksum(const std::string &name, const std::string &checksum)
Normalize checksum.
static Status ProcessConfig(std::map< std::string, std::string > &config, const std::string &file)
Process a config file and return key-value pairs.
static Status ProcessConfigDir(std::map< std::string, std::string > &config, const std::string &dir)
Process a config directory and return key-value pairs.
static std::string FQDNToCC(const std::string &fqdn)
Convert the fully qualified host name to country code.
static std::string InferChecksumType(const XrdCl::URL &source, const XrdCl::URL &destination, bool zip=false)
Automatically infer the right checksum type.
static void LogPropertyList(Log *log, uint64_t topic, const char *format, const PropertyList &list)
Log property list.
static std::string Char2Hex(uint8_t *array, uint16_t size)
Print a char array as hex.
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
static Status GetHostAddresses(std::vector< XrdNetAddr > &addresses, const URL &url, AddressType type)
Resolve IP addresses.
static uint64_t GetElapsedMicroSecs(timeval start, timeval end)
Get the elapsed microseconds between two timevals.
static std::vector< std::string > GetSupportedCheckSums(const XrdCl::URL &url)
Get supported checksum types for given URL.
static AddressType String2AddressType(const std::string &addressType)
Interpret a string as address type, default to IPAll.
static int GetIntParameter(const URL &url, const std::string &name, int defaultVal)
Get a parameter either from the environment or URL.
static Status GetDirectoryEntries(std::vector< std::string > &entries, const std::string &path)
Get directory entries.
static XRootDStatus GetLocalCheckSum(std::string &checkSum, const std::string &checkSumType, const std::string &path)
Get a checksum from local file.
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
static void Trim(std::string &str)
Trim a string.
static bool CheckEC(const Message *req, const URL &url)
Check if this client can support given EC redirect.
static XRootDStatus GetRemoteCheckSum(std::string &checkSum, const std::string &checkSumType, const URL &url)
Get a checksum from a remote xrootd server.
static std::string GetStringParameter(const URL &url, const std::string &name, const std::string &defaultVal)
Get a parameter either from the environment or URL.
static XRootDStatus CheckTPC(const std::string &server, uint16_t timeout=0)
Check if peer supports tpc.
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
An interface for metadata redirectors.
virtual std::vector< std::string > GetSupportedCheckSums() const =0
const std::string & GetErrorMessage() const
Get error message.
void SetErrorMessage(const std::string &message)
Set the error message.
std::string ToStr() const
Convert to string.
bool isIPType(IPType ipType) const
static const char * GetAddrs(const char *hSpec, XrdNetAddr *aListP[], int &aListN, AddrOpts opts=allIPMap, int pNum=PortInSpec)
const uint16_t errInvalidAddr
const uint16_t stFatal
Fatal error, it's still an error.
const uint16_t stError
An error occurred that could potentially be retried.
const uint16_t errInternal
Internal error.
const uint16_t stOK
Everything went OK.
const int DefaultIPNoShuffle
const uint16_t errConfig
System misconfigured.
const uint16_t errOSError
const uint64_t UtilityMsg
const uint16_t errInvalidResponse
std::vector< ChunkInfo > ChunkList
List of chunks.
const uint16_t errNotSupported
const int DefaultPreferIPv4
const uint16_t errCheckSumError
const int DefaultZipMtlnCksum
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ Config
Query server configuration.
@ Checksum
Query file checksum.
Procedure execution status.
uint16_t status
Status of the execution.
bool IsOK() const
We're fine.
std::string ToString() const
Create a string representation.