xrootd
XrdCmsCluster.hh
Go to the documentation of this file.
1 #ifndef __CMS_CLUSTER__H
2 #define __CMS_CLUSTER__H
3 /******************************************************************************/
4 /* */
5 /* X r d C m s C l u s t e r . h h */
6 /* */
7 /* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */
8 /* All Rights Reserved */
9 /* Produced by Andrew Hanushevsky for Stanford University under contract */
10 /* DE-AC02-76-SFO0515 with the Department of Energy */
11 /* */
12 /* This file is part of the XRootD software suite. */
13 /* */
14 /* XRootD is free software: you can redistribute it and/or modify it under */
15 /* the terms of the GNU Lesser General Public License as published by the */
16 /* Free Software Foundation, either version 3 of the License, or (at your */
17 /* option) any later version. */
18 /* */
19 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
20 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
21 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
22 /* License for more details. */
23 /* */
24 /* You should have received a copy of the GNU Lesser General Public License */
25 /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
26 /* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
27 /* */
28 /* The copyright holder's institutional names and contributor's names may not */
29 /* be used to endorse or promote products derived from this software without */
30 /* specific prior written permission of the institution or contributor. */
31 /******************************************************************************/
32 
33 #include <stdlib.h>
34 #include <string.h>
35 #include <strings.h>
36 #include <netinet/in.h>
37 
38 #include "XrdCms/XrdCmsTypes.hh"
39 #include "XrdOuc/XrdOucTList.hh"
40 #include "XrdOuc/XrdOucEnum.hh"
41 #include "XrdSys/XrdSysPthread.hh"
42 
43 class XrdLink;
44 class XrdCmsDrop;
45 class XrdCmsNode;
46 class XrdCmsSelect;
47 class XrdCmsSelector;
48 class XrdNetAddr;
49 
50 namespace XrdCms
51 {
52 struct CmsRRHdr;
53 }
54 
55 /******************************************************************************/
56 /* O p t i o n F l a g s */
57 /******************************************************************************/
58 
59 namespace XrdCms
60 {
61 
62 // Flags passed to Add()
63 //
64 static const int CMS_noStage = 1;
65 static const int CMS_Suspend = 2;
66 static const int CMS_Perm = 4;
67 static const int CMS_isMan = 8;
68 static const int CMS_Lost = 16;
69 static const int CMS_isPeer = 32;
70 static const int CMS_isProxy = 64;
71 static const int CMS_noSpace =128;
72 static const int CMS_isSuper =256;
73 
74 static const int CMS_isVers3 =0x01000000;
75 
77 static const int CMS_hasAlts =CMS_isMan|CMS_isPeer;
78 
79 // Class passed to Space()
80 //
81 class SpaceData
82 {
83 public:
84 
85 long long Total; // Total space
86 long long TotFr; // Total space free
87 int wMinF; // Free space minimum to select wFree node
88 int wFree; // Free space for nodes providing r/w access (largest one)
89 int wNum; // Number of nodes providing r/w access
90 int wUtil; // Average utilization (largest one)
91 int sFree; // Free space for nodes providing staging (largest one)
92 int sNum; // Number of nodes providing staging
93 int sUtil; // Average utilization (largest one)
94 
95  SpaceData() : Total(0), TotFr(0),wMinF(0),
96  wFree(0), wNum(0), wUtil(0),
97  sFree(0), sNum(0), sUtil(0) {}
99 };
100 }
101 
102 /******************************************************************************/
103 /* C l a s s X r d C m s C l u s t e r */
104 /******************************************************************************/
105 
106 // This a single-instance global class
107 //
108 class XrdCmsBaseFR;
109 class XrdCmsClustID;
110 class XrdCmsSelected;
111 class XrdOucTList;
112 
114 {
115 public:
116 friend class XrdCmsDrop;
117 
118 int NodeCnt; // Number of active nodes
119 
120 // Called to add a new node to the cluster. Status values are defined above.
121 //
122 XrdCmsNode *Add(XrdLink *lp, int dport, int Status,
123  int sport, const char *theNID, const char *theIF);
124 
125 // Put nodes in or remove from a blacklist
126 //
127 virtual void BlackList(XrdOucTList *blP);
128 
129 // Sends a message to all nodes matching smask (three forms for convenience)
130 //
131 SMask_t Broadcast(SMask_t, const struct iovec *, int, int tot=0);
132 
134  char *Data, int Dlen=0);
135 
137  void *Data, int Dlen);
138 
139 // Sends a message to a single node in a round-robbin fashion.
140 //
142  void *Data, int Dlen);
143 
144 // Returns the node mask matching the given IP address
145 //
147 
148 // Returns the node mask matching the given cluster ID
149 //
150 SMask_t getMask(const char *Cid);
151 
152 // Extracts out node information. Opts are one or more of CmsLSOpts
153 //
154 enum CmsLSOpts {LS_NULL=0, LS_IPO=0x0100, LS_IDNT=0x0200,
155  LS_ANY =0x0400, LS_IFMASK = 0x0f};
156 
157 XrdCmsSelected *List(SMask_t mask, CmsLSOpts opts, bool &oksel);
158 
159 // Returns the location of a file
160 //
162 
163 // Always run as a separate thread to monitor subscribed node performance
164 //
165 void *MonPerf();
166 
167 // Alwats run as a separate thread to maintain the node reference count
168 //
169 void *MonRefs();
170 
171 // Return total number of redirect references (sloppy as we don't lock it)
172 //
173 long long Refs() {return SelWcnt+SelWtot+SelRcnt+SelRtot;}
174 
175 // Called to remove a node from the cluster
176 //
177 void Remove(XrdCmsNode *theNode);
178 
179 void Remove(const char *reason, XrdCmsNode *theNode, int immed=0);
180 
181 // Called to reset the node reference counts for nodes matching smask
182 //
183 void ResetRef(SMask_t smask, bool isLocked=false);
184 
185 // Called to select the best possible node to serve a file (two forms)
186 //
187 static const int NotFound = -1; // Locate() failed to find resource
188 static const int Wait4CBk = -2; // Only returned by Locate()
189 static const int RetryErr = -3; // Used only by XrdCmsNode retry processing
190 static const int EReplete = -4; // Failed, error information is complete
191 
193 
194 int Select(SMask_t pmask, int &port, char *hbuff, int &hlen,
195  int isrw, int isMulti, int ifWant);
196 
197 // Manipulate the global selection lock
198 //
199 void SLock(bool dolock)
200  {if (dolock) STMutex.Lock();
201  else STMutex.UnLock();
202  }
203 
204 // Called to get cluster space (for managers and supervisors only)
205 //
206 void Space(XrdCms::SpaceData &sData, SMask_t smask);
207 
208 // Called to return statistics
209 //
210 int Stats(char *bfr, int bln); // Server
211 int Statt(char *bfr, int bln); // Manager
212 
214 virtual ~XrdCmsCluster() {} // This object should never be deleted
215 
216 private:
217 XrdCmsNode *AddAlt(XrdCmsClustID *cidP, XrdLink *lp, int port, int Status,
218  int sport, const char *theNID, const char *theIF);
220 int Drop(int sent, int sinst, XrdCmsDrop *djp=0);
221 void Record(char *path, const char *reason, bool force=false);
222 bool maxBits(SMask_t mVec, int mbits);
223 int Multiple(SMask_t mVec);
224 enum {eExists, eDups, eROfs, eNoRep, eNoSel, eNoEnt}; // Passed to SelFail
225 int SelFail(XrdCmsSelect &Sel, int rc);
226 int SelNode(XrdCmsSelect &Sel, SMask_t pmask, SMask_t amask);
230 int SelDFS(XrdCmsSelect &Sel, SMask_t amask,
231  SMask_t &pmask, SMask_t &smask, int isRW);
232 void sendAList(XrdLink *lp);
233 void setAltMan(int snum, XrdLink *lp, int port);
234 int Unreachable(XrdCmsSelect &Sel, bool none);
236 
237 // Number of <host>:Port characters per entry was INET6_ADDRSTRLEN+10
238 //
239 static const int AltSize = 254; // We may revert to IP address
240 
241 XrdSysMutex XXMutex; // Protects cluster summary state variables
242 XrdSysMutex STMutex; // Protects all node information variables
243 XrdCmsNode *NodeTab[STMax]; // Current set of nodes
244 
245 int STHi; // NodeTab high watermark
247 long long SelWcnt; // Curr number of r/w selections (successful)
248 long long SelWtot; // Total number of r/w selections (successful)
249 long long SelRcnt; // Curr number of r/o selections (successful)
250 long long SelRtot; // Total number of r/o selections (successful)
251 long long SelTcnt; // Total number of all selections
252 
253 // The following is a list of IP:Port tokens that identify supervisor nodes.
254 // The information is sent via the try request to redirect nodes; as needed.
255 // The list is alays rotated by one entry each time it is sent.
256 //
257 char AltMans[STMax*AltSize]; // ||123.123.123.123:12345|| = 21
258 char *AltMend;
260 
261 // The foloowing three variables are protected by the STMutex
262 //
263 SMask_t peerHost; // Nodes that are acting as peers
264 SMask_t peerMask; // Always ~peerHost
265 };
266 
268 
269 namespace XrdCms
270 {
271 extern XrdCmsCluster Cluster;
272 }
273 #endif
XrdCmsCluster::Space
void Space(XrdCms::SpaceData &sData, SMask_t smask)
XrdCmsCluster::EReplete
static const int EReplete
Definition: XrdCmsCluster.hh:190
XrdCms::SpaceData::SpaceData
SpaceData()
Definition: XrdCmsCluster.hh:95
XrdCmsCluster::SelbyRef
XrdCmsNode * SelbyRef(SMask_t, XrdCmsSelector &selR)
XrdCms::SpaceData::wNum
int wNum
Definition: XrdCmsCluster.hh:89
XrdCmsCluster::Record
void Record(char *path, const char *reason, bool force=false)
XrdCmsCluster::peerMask
SMask_t peerMask
Definition: XrdCmsCluster.hh:264
XrdCmsCluster::AddAlt
XrdCmsNode * AddAlt(XrdCmsClustID *cidP, XrdLink *lp, int port, int Status, int sport, const char *theNID, const char *theIF)
XrdCmsCluster::Remove
void Remove(const char *reason, XrdCmsNode *theNode, int immed=0)
XrdCms::SpaceData::TotFr
long long TotFr
Definition: XrdCmsCluster.hh:86
STMax
#define STMax
Definition: XrdCmsTypes.hh:39
XrdCmsCluster::eNoSel
@ eNoSel
Definition: XrdCmsCluster.hh:224
XrdCmsCluster::List
XrdCmsSelected * List(SMask_t mask, CmsLSOpts opts, bool &oksel)
XrdCmsCluster::LS_NULL
@ LS_NULL
Definition: XrdCmsCluster.hh:154
XrdCms::SpaceData::~SpaceData
~SpaceData()
Definition: XrdCmsCluster.hh:98
XrdCmsSelect
Definition: XrdCmsSelect.hh:44
SMask_t
unsigned long long SMask_t
Definition: XrdCmsTypes.hh:33
XrdSysMutex
Definition: XrdSysPthread.hh:166
XrdCms::SpaceData::wMinF
int wMinF
Definition: XrdCmsCluster.hh:87
XrdCmsCluster::Unuseable
int Unuseable(XrdCmsSelect &Sel)
XrdSysPthread.hh
XrdCmsCluster::Unreachable
int Unreachable(XrdCmsSelect &Sel, bool none)
XrdCmsCluster::LS_IFMASK
@ LS_IFMASK
Definition: XrdCmsCluster.hh:155
XrdCmsCluster::AltMans
char AltMans[STMax *AltSize]
Definition: XrdCmsCluster.hh:257
XrdCms::CMS_hasAlts
static const int CMS_hasAlts
Definition: XrdCmsCluster.hh:77
XrdCmsCluster::LS_ANY
@ LS_ANY
Definition: XrdCmsCluster.hh:155
XrdCmsCluster::Stats
int Stats(char *bfr, int bln)
XrdCms::CMS_Perm
static const int CMS_Perm
Definition: XrdCmsCluster.hh:66
XrdCmsCluster::AltMend
char * AltMend
Definition: XrdCmsCluster.hh:258
XrdCms::SpaceData
Definition: XrdCmsCluster.hh:82
XrdCmsCluster::Select
int Select(SMask_t pmask, int &port, char *hbuff, int &hlen, int isrw, int isMulti, int ifWant)
XrdCmsCluster::SelTcnt
long long SelTcnt
Definition: XrdCmsCluster.hh:251
XrdCmsCluster::MonPerf
void * MonPerf()
XrdCmsCluster::Broadsend
int Broadsend(SMask_t smask, XrdCms::CmsRRHdr &Hdr, void *Data, int Dlen)
XrdCmsCluster::MonRefs
void * MonRefs()
XrdCmsCluster::maxBits
bool maxBits(SMask_t mVec, int mbits)
XrdCms::CMS_noStage
static const int CMS_noStage
Definition: XrdCmsCluster.hh:64
XrdCms::CMS_isPeer
static const int CMS_isPeer
Definition: XrdCmsCluster.hh:69
XrdCmsCluster::getMask
SMask_t getMask(const char *Cid)
XrdCmsNode
Definition: XrdCmsNode.hh:57
XrdCmsCluster::SelFail
int SelFail(XrdCmsSelect &Sel, int rc)
XrdCms::SpaceData::wFree
int wFree
Definition: XrdCmsCluster.hh:88
XrdOucEnum.hh
XrdCmsCluster::ResetRef
void ResetRef(SMask_t smask, bool isLocked=false)
XrdCmsCluster::sendAList
void sendAList(XrdLink *lp)
XrdCmsCluster::STHi
int STHi
Definition: XrdCmsCluster.hh:245
XrdCmsCluster::XrdCmsCluster
XrdCmsCluster()
XrdCmsCluster::NotFound
static const int NotFound
Definition: XrdCmsCluster.hh:187
XrdCmsCluster::eNoEnt
@ eNoEnt
Definition: XrdCmsCluster.hh:224
XrdCmsCluster::SelNode
int SelNode(XrdCmsSelect &Sel, SMask_t pmask, SMask_t amask)
XrdCmsCluster::SelRcnt
long long SelRcnt
Definition: XrdCmsCluster.hh:249
XrdCmsCluster
Definition: XrdCmsCluster.hh:114
XrdCms::CmsRRHdr
Definition: YProtocol.hh:83
XrdCms::CMS_isSuper
static const int CMS_isSuper
Definition: XrdCmsCluster.hh:72
XrdCmsCluster::Broadcast
SMask_t Broadcast(SMask_t, const struct iovec *, int, int tot=0)
XrdCmsCluster::getMask
SMask_t getMask(const XrdNetAddr *addr)
XrdCms
Definition: YProtocol.hh:78
XrdCmsCluster::SelDFS
int SelDFS(XrdCmsSelect &Sel, SMask_t amask, SMask_t &pmask, SMask_t &smask, int isRW)
XrdCmsCluster::SelbyLoad
XrdCmsNode * SelbyLoad(SMask_t, XrdCmsSelector &selR)
XRDOUC_ENUM_OPERATORS
#define XRDOUC_ENUM_OPERATORS(T)
Definition: XrdOucEnum.hh:22
XrdCmsCluster::calcDelay
XrdCmsNode * calcDelay(XrdCmsSelector &selR)
XrdCmsCluster::Statt
int Statt(char *bfr, int bln)
XrdCmsCluster::XrdCmsDrop
friend class XrdCmsDrop
Definition: XrdCmsCluster.hh:116
XrdCmsCluster::eNoRep
@ eNoRep
Definition: XrdCmsCluster.hh:224
XrdCmsCluster::eROfs
@ eROfs
Definition: XrdCmsCluster.hh:224
XrdCmsCluster::setAltMan
void setAltMan(int snum, XrdLink *lp, int port)
XrdCms::CMS_isMan
static const int CMS_isMan
Definition: XrdCmsCluster.hh:67
XrdCmsBaseFR
Definition: XrdCmsBaseFS.hh:49
XrdCmsCluster::SelRtot
long long SelRtot
Definition: XrdCmsCluster.hh:250
XrdCmsClustID
Definition: XrdCmsClustID.hh:41
XrdCmsCluster::LS_IDNT
@ LS_IDNT
Definition: XrdCmsCluster.hh:154
XrdCms::CMS_isVers3
static const int CMS_isVers3
Definition: XrdCmsCluster.hh:74
XrdCmsCluster::SLock
void SLock(bool dolock)
Definition: XrdCmsCluster.hh:199
XrdCmsCluster::AltSize
static const int AltSize
Definition: XrdCmsCluster.hh:239
XrdCmsCluster::eExists
@ eExists
Definition: XrdCmsCluster.hh:224
XrdCmsSelected
Definition: XrdCmsSelect.hh:102
XrdCmsCluster::Broadcast
SMask_t Broadcast(SMask_t smask, XrdCms::CmsRRHdr &Hdr, void *Data, int Dlen)
XrdCmsCluster::SelWcnt
long long SelWcnt
Definition: XrdCmsCluster.hh:247
XrdCmsCluster::Broadcast
SMask_t Broadcast(SMask_t smask, XrdCms::CmsRRHdr &Hdr, char *Data, int Dlen=0)
XrdCms::CMS_Lost
static const int CMS_Lost
Definition: XrdCmsCluster.hh:68
XrdCmsCluster::NodeTab
XrdCmsNode * NodeTab[STMax]
Definition: XrdCmsCluster.hh:243
XrdCmsCluster::Remove
void Remove(XrdCmsNode *theNode)
XrdCmsCluster::Locate
int Locate(XrdCmsSelect &Sel)
XrdCmsCluster::Drop
int Drop(int sent, int sinst, XrdCmsDrop *djp=0)
XrdCms::CMS_isProxy
static const int CMS_isProxy
Definition: XrdCmsCluster.hh:70
XrdCms::CMS_noSpace
static const int CMS_noSpace
Definition: XrdCmsCluster.hh:71
XrdCms::SpaceData::sUtil
int sUtil
Definition: XrdCmsCluster.hh:93
XrdCmsCluster::Select
int Select(XrdCmsSelect &Sel)
XrdSysMutex::UnLock
void UnLock()
Definition: XrdSysPthread.hh:224
XrdCl::none
static struct XrdCl::None none
XrdCmsCluster::Add
XrdCmsNode * Add(XrdLink *lp, int dport, int Status, int sport, const char *theNID, const char *theIF)
XrdCmsCluster::LS_IPO
@ LS_IPO
Definition: XrdCmsCluster.hh:154
XrdCmsCluster::RetryErr
static const int RetryErr
Definition: XrdCmsCluster.hh:189
XrdCmsCluster::SelWtot
long long SelWtot
Definition: XrdCmsCluster.hh:248
XrdSysMutex::Lock
void Lock()
Definition: XrdSysPthread.hh:222
XrdCmsSelector
Definition: XrdCmsSelect.hh:139
XrdCmsCluster::eDups
@ eDups
Definition: XrdCmsCluster.hh:224
XrdCmsCluster::Refs
long long Refs()
Definition: XrdCmsCluster.hh:173
XrdCms::SpaceData::Total
long long Total
Definition: XrdCmsCluster.hh:85
XrdCmsCluster::peerHost
SMask_t peerHost
Definition: XrdCmsCluster.hh:263
XrdCmsCluster::BlackList
virtual void BlackList(XrdOucTList *blP)
XrdCmsCluster::STMutex
XrdSysMutex STMutex
Definition: XrdCmsCluster.hh:242
XrdCmsCluster::Multiple
int Multiple(SMask_t mVec)
XrdCms::SpaceData::sNum
int sNum
Definition: XrdCmsCluster.hh:92
XrdCmsCluster::XXMutex
XrdSysMutex XXMutex
Definition: XrdCmsCluster.hh:241
XrdCms::CMS_notServ
static const int CMS_notServ
Definition: XrdCmsCluster.hh:76
XrdCms::Cluster
XrdCmsCluster Cluster
XrdOucTList.hh
XrdCms::SpaceData::wUtil
int wUtil
Definition: XrdCmsCluster.hh:90
XrdCmsTypes.hh
XrdNetAddr
Definition: XrdNetAddr.hh:42
XrdCmsCluster::Reserved
int Reserved
Definition: XrdCmsCluster.hh:246
XrdCms::CMS_Suspend
static const int CMS_Suspend
Definition: XrdCmsCluster.hh:65
XrdCms::SpaceData::sFree
int sFree
Definition: XrdCmsCluster.hh:91
XrdOucTList
Definition: XrdOucTList.hh:42
XrdCmsCluster::NodeCnt
int NodeCnt
Definition: XrdCmsCluster.hh:118
XrdCmsCluster::SelbyCost
XrdCmsNode * SelbyCost(SMask_t, XrdCmsSelector &selR)
XrdCmsCluster::AltMent
int AltMent
Definition: XrdCmsCluster.hh:259
XrdCmsCluster::Wait4CBk
static const int Wait4CBk
Definition: XrdCmsCluster.hh:188
XrdCmsCluster::~XrdCmsCluster
virtual ~XrdCmsCluster()
Definition: XrdCmsCluster.hh:214
XrdCmsCluster::CmsLSOpts
CmsLSOpts
Definition: XrdCmsCluster.hh:154