Main MRPT website > C++ reference for MRPT 1.5.6
net_utils.cpp
Go to the documentation of this file.
1 /* +---------------------------------------------------------------------------+
2  | Mobile Robot Programming Toolkit (MRPT) |
3  | http://www.mrpt.org/ |
4  | |
5  | Copyright (c) 2005-2017, Individual contributors, see AUTHORS file |
6  | See: http://www.mrpt.org/Authors - All rights reserved. |
7  | Released under BSD License. See details in http://www.mrpt.org/License |
8  +---------------------------------------------------------------------------+ */
9 
10 #include "base-precomp.h" // Precompiled headers
11 
12 #include <mrpt/utils/net_utils.h>
15 #include <mrpt/utils/CTicTac.h>
17 #include <mrpt/system/threads.h>
18 #include <mrpt/synch/CSemaphore.h>
20 #include <cstring>
21 #include <stdio.h>
22 
23 #if defined(MRPT_OS_LINUX) || defined(MRPT_OS_APPLE)
24  #define INVALID_SOCKET (-1)
25  #include <sys/socket.h>
26  #include <unistd.h>
27  #include <fcntl.h>
28  #include <errno.h>
29  #include <sys/types.h>
30  #include <sys/ioctl.h>
31  #include <netdb.h>
32  #include <arpa/inet.h>
33  #include <netinet/in.h>
34 #endif
35 
36 #ifdef MRPT_OS_WINDOWS
37  #include <winsock.h>
38 #endif
39 
40 
41 using namespace mrpt;
42 using namespace mrpt::synch;
43 using namespace mrpt::system;
44 using namespace mrpt::utils;
45 using namespace mrpt::utils::net;
46 using namespace std;
47 
48 /*---------------------------------------------------------------
49  http_get
50  ---------------------------------------------------------------*/
53  const string &url,
54  string &out_content,
55  string &out_errormsg,
56  int port,
57  const string &auth_user,
58  const string &auth_pass,
59  int *out_http_responsecode,
60  mrpt::utils::TParameters<string> *extra_headers,
62  int timeout_ms
63  )
64 {
66  ERRORCODE_HTTP ret = http_get(url,data,out_errormsg,port,auth_user,auth_pass,out_http_responsecode,extra_headers,out_headers,timeout_ms);
67 
68 
69  out_content.resize(data.size());
70  if (!data.empty())
71  ::memcpy(&out_content[0],&data[0],data.size());
72 
73  return ret;
74 }
75 
76 
79  const string & http_method,
80  const string & http_send_content,
81  const string &url,
82  vector_byte &out_content,
83  string &out_errormsg,
84  int port,
85  const string &auth_user,
86  const string &auth_pass,
87  int *out_http_responsecode,
88  mrpt::utils::TParameters<string> *extra_headers,
90  int timeout_ms
91  )
92 {
93  // Reset output data:
94  out_content.clear();
95  if (out_http_responsecode) *out_http_responsecode=0;
96  if (out_headers) out_headers->clear();
97 
98  // URL must be:
99  // http://<SERVER>/<LOCAL_ADDR>
100 
101  if (0!=::strncmp(url.c_str(),"http://",7))
102  {
103  out_errormsg="URL must start with 'http://'";
104  return net::erBadURL;
105  }
106 
107  string server_addr = url.substr(7);
108  string get_object = "/";
109 
110  // Remove from the first "/" on:
111  size_t pos = server_addr.find("/");
112  if (pos==0)
113  {
114  out_errormsg="Server name not found in URL";
115  return net::erBadURL;
116  }
117  if (pos!=string::npos)
118  {
119  get_object = server_addr.substr(pos);
120  server_addr.resize(pos);
121  }
122 
123  CClientTCPSocket sock;
124 
125  try {
126  // Connect:
127  sock.connect(server_addr,port, timeout_ms);
128  }
129  catch (std::exception &e) {
130  out_errormsg = e.what();
131  return net::erCouldntConnect;
132  }
133 
134  try {
135  // Set the user-defined headers (we may overwrite them if needed)
136  TParameters<string> headers_to_send;
137  if (extra_headers)
138  headers_to_send=*extra_headers;
139 
140  headers_to_send["Connection"]="close"; // Don't keep alive
141 
142  if (!headers_to_send.has("User-Agent"))
143  headers_to_send["User-Agent"]="MRPT Library";
144 
145  // Implement HTTP Basic authentication:
146  // See: http://en.wikipedia.org/wiki/Basic_access_authentication
147  if (!auth_user.empty())
148  {
149  string auth_str = auth_user+string(":")+auth_pass;
150  vector_byte v(auth_str.size());
151  ::memcpy(&v[0],&auth_str [0],auth_str .size());
152 
153  string encoded_str;
154  mrpt::system::encodeBase64(v,encoded_str);
155 
156  headers_to_send["Authorization"]=string("Basic ")+encoded_str;
157  }
158 
159  if (!http_send_content.empty() && headers_to_send.find("Content-Length") == headers_to_send.end())
160  {
161  headers_to_send["Content-Length"] = mrpt::format("%u", (unsigned int)http_send_content.size());
162  }
163 
164  // Prepare the request string
165  // ---------------------------------
166  string req = format(
167  "%s %s HTTP/1.1\r\n"
168  "Host: %s\r\n",
169  http_method.c_str(),
170  get_object.c_str(),server_addr.c_str());
171 
172  // Other headers:
173  for (TParameters<string>::const_iterator i=headers_to_send.begin();i!=headers_to_send.end();++i)
174  {
175  req+=i->first;
176  req+=": ";
177  req+=i->second;
178  req+="\r\n";
179  }
180 
181  // End:
182  req+="\r\n";
183 
184  // Any POST data?
185  req += http_send_content;
186 
187  // Send:
188  sock.sendString(req);
189 
190  // Read answer:
191  vector_byte buf;
192  buf.reserve(1<<14);
193 
194  size_t total_read = 0;
195  bool content_length_read = false;
196  bool all_headers_read = false;
197  size_t content_length = 0;
198  size_t content_offset = 0;
199  int http_code = 0;
201 
202  CTicTac watchdog;
203  watchdog.Tic();
204 
205  while (!content_length_read || total_read<(content_offset+content_length))
206  {
207  // Read until "Content-Length: XXX \r\n" is read or the whole message is read,
208  // or an error code is read.
209  size_t to_read_now;
210 
211  if (!content_length_read)
212  {
213  to_read_now = 1500;
214  }
215  else
216  {
217  to_read_now = (content_length+content_offset) -total_read;
218  }
219 
220  // make room for the data to come:
221  buf.resize(total_read+to_read_now+1);
222 
223  // Read:
224  size_t len = sock.readAsync(&buf[total_read],to_read_now, timeout_ms,100);
225  if (!len)
226  {
227  //
228  if (!sock.isConnected())
229  {
230  if (all_headers_read) // It seems we're done...
231  break;
232  else
233  {
234  out_errormsg = "Connection to server was lost";
235  return net::erCouldntConnect;
236  }
237  }
238 
239  if ( watchdog.Tac()>1e-3*timeout_ms)
240  {
241  out_errormsg = "Timeout waiting answer from server";
242  return net::erCouldntConnect;
243  }
245  continue;
246  }
247  total_read+=len;
248  watchdog.Tic();
249 
250  buf[total_read]='\0';
251 
252  // do we have a \r\n\r\n ??
253  if (!all_headers_read)
254  {
255  const char *ptr = ::strstr(reinterpret_cast<const char*>(&buf[0]),"\r\n\r\n");
256  if (ptr)
257  {
258  all_headers_read = true;
259  const size_t pos_dblret = ((char*)ptr)-(char*)(&buf[0]);
260 
261  // Process the headers:
262  // ------------------------------
263  if (!::strncmp("HTTP/",(const char*)&buf[0], 5))
264  {
265  http_code = ::atoi( (const char*)&buf[9] );
266  }
267  else
268  {
269  // May it be a "SOURCETABLE " answer for NTRIP protocol??
270  if (!::strncmp("SOURCETABLE ",(const char*)&buf[0], 12))
271  {
272  http_code = ::atoi( (const char*)&buf[12] );
273  }
274  else
275  {
276  out_errormsg = "Server didn't send an HTTP/1.1 answer.";
277  return net::erOtherHTTPError;
278  }
279  }
280 
281  // Check the HTTP code and the content-length:
282  content_offset = pos_dblret + 4;
283 
284  // Do we have a "Content-Length:"??
285  const char *ptr_len = ::strstr(reinterpret_cast<const char*>(&buf[0]),"Content-Length:");
286  if (ptr_len)
287  {
288  content_length = ::atol( ptr_len+15 );
289  content_length_read = true;
290  }
291 
292  // Parse the rest of HTTP headers:
293  {
294  string aux_all_headers;
295  deque<string> lstLines;
296  aux_all_headers.resize(content_offset);
297  ::memcpy( &aux_all_headers[0],&buf[0],content_offset);
298 
299  mrpt::system::tokenize(aux_all_headers,"\r\n",lstLines);
300 
301  for (deque<string>::const_iterator i=lstLines.begin();i!=lstLines.end();++i)
302  {
303  const size_t p = i->find(":");
304  if (p==string::npos) continue;
305 
306  const string key = i->substr(0,p);
307  const string val = i->substr(p+2);
308  rx_headers[key] = val;
309  }
310  }
311 
312  }
313  }
314  } // end while
315 
316 
317  if (out_http_responsecode) *out_http_responsecode=http_code;
318  if (out_headers) *out_headers = rx_headers;
319 
320  // Remove the headers from the content:
321  buf.erase(buf.begin(),buf.begin()+content_offset);
322 
323  // Process: "Transfer-Encoding: chunked"
324  if (rx_headers.has("Transfer-Encoding") && rx_headers["Transfer-Encoding"]=="chunked" )
325  {
326  // See: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
327 
328  size_t index = 0;
329  while (index<buf.size())
330  {
331  if (buf[index]=='\r' && buf[index+1]=='\n')
332  {
333  buf.erase(buf.begin()+index,buf.begin()+index+2);
334  continue;
335  }
336 
337  const char *pCRLF = ::strstr((const char*)&buf[index],"\r\n");
338  if (!pCRLF) break;
339 
340  const size_t len_substr = ((char*)pCRLF)-(char*)(&buf[index]);
341 
342  string sLen((const char*)&buf[index], len_substr);
343  sLen=string("0x")+sLen;
344 
345  unsigned int lenChunk;
346  int fields = ::sscanf(sLen.c_str(),"%x",&lenChunk);
347  if (!fields) break;
348 
349  // Remove the len of this chunk header from the data:
350  buf.erase(buf.begin()+index,buf.begin()+index+len_substr+2);
351 
352  index+=lenChunk;
353 
354  if (!lenChunk)
355  {
356  buf.resize(index);
357  break;
358  }
359  }
360  }
361 
362  // Set the content output:
363  out_content.swap(buf);
364 
365  if (http_code==200)
366  {
367  return net::erOk;
368  }
369  else
370  {
371  out_errormsg = format("HTTP error %i",http_code);
372  return net::erOtherHTTPError;
373  }
374  }
375  catch (std::exception &e) {
376  out_errormsg = e.what();
377  return net::erCouldntConnect;
378  }
379 
380  return net::erOk;
381 }
382 
383 
384 /*---------------------------------------------------------------
385  http_get
386 ---------------------------------------------------------------*/
389  const string &url,
390  vector_byte &out_content,
391  string &out_errormsg,
392  int port,
393  const string &auth_user,
394  const string &auth_pass,
395  int *out_http_responsecode,
396  mrpt::utils::TParameters<string> *extra_headers,
398  int timeout_ms
399  )
400 {
401  return http_request("GET", "", url, out_content, out_errormsg, port, auth_user, auth_pass, out_http_responsecode, extra_headers, out_headers, timeout_ms);
402 }
403 
404 
405 // =========== net::DNS_resolve_async ==============
406 // Yeah, it's gonna got pretty complicated...
408 {
410  sem_solved(0,1),
411  sem_caller_quitted(0,1),
412  in_servername(),
413  out_solved_ip()
414  {}
416  synch::CSemaphore sem_caller_quitted; // This is needed in order to the thread to wait for deleting this struct!
418  CThreadSafeVariable<std::string> out_solved_ip; // Will be set to NULL by the caller if the function returns.
419 };
420 
421 void thread_DNS_solver_async(TDNSThreadData &dat); // Frd decl.
422 
423 
424 
425 /** Resolve a server address by its name, returning its IP address as a string - This method has a timeout for the maximum time to wait for the DNS server.
426  * For example: server_name="www.google.com" -> out_ip="209.85.227.99"
427  *
428  * \return true on success, false on timeout or other error.
429  */
431  const std::string &server_name,
432  std::string &out_ip,
433  const unsigned int timeout_ms
434  )
435 {
436  // Firstly: If it's a numeric address already, do nothing:
437  if (server_name.find_first_not_of("0123456789. ")==std::string::npos)
438  {
439  // It's a pure IP address:
440  out_ip = server_name;
441  return true;
442  }
443 
444  // Solve DNS --------------
445  // It seems that the only reliable way of *with a timeout* is to launch a separate thread.
447  param.in_servername = server_name;
448 
449  TThreadHandle th =
451 
452  bool res =(param.sem_solved.waitForSignal(timeout_ms));
453  // Let the thread now about me quitting:
454  param.sem_caller_quitted.release();
456 
457  if (res)
458  {
459  // Done: Anyway, it can still be an error result:
460  out_ip = param.out_solved_ip;
461  return !out_ip.empty();
462  }
463  else
464  {
465  // Timeout:
466  out_ip.clear();
467 
468  return false;
469  }
470 }
471 
473 {
474  // Windows-specific stuff:
475 #ifdef MRPT_OS_WINDOWS
476  {
477  // Init the WinSock Library:
478  WORD wVersionRequested = MAKEWORD( 2, 0 );
479  WSADATA wsaData;
480  if (WSAStartup( wVersionRequested, &wsaData ) )
481  {
482  std::cerr << "thread_DNS_solver_async: Error calling WSAStartup";
483  return;
484  }
485  }
486 #endif
487 
488  // Do the DNS lookup:
489  std::string dns_result;
490 
491  hostent *he = gethostbyname( param.in_servername.c_str() );
492  if (!he)
493  {
494  dns_result.clear(); // empty string -> error.
495  }
496  else
497  {
498  struct in_addr ADDR;
499  ::memcpy(&ADDR, he->h_addr, sizeof(ADDR)); // Was: *((struct in_addr *)he->h_addr);
500  // Convert address to text:
501  dns_result = string( inet_ntoa(ADDR) );
502  }
503 
504  // Save in the caller string:
505  param.out_solved_ip.set( dns_result );
506 
507  // and signal we're done:
508  param.sem_solved.release();
509 
510  // Finally, wait for the main thread to end so we can safely free the shared struct:
511  param.sem_caller_quitted.waitForSignal();
512 
513 #ifdef MRPT_OS_WINDOWS
514  WSACleanup();
515 #endif
516 }
517 
518 /** Returns a description of the last Sockets error */
520 {
521 #ifdef MRPT_OS_WINDOWS
522  const int errnum = WSAGetLastError();
523  char * s = NULL;
524  FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
525  NULL, errnum,
526  MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
527  (LPSTR)&s, 0, NULL);
528  const std::string str = mrpt::format("%s [errno=%d]",s,errnum);
529  LocalFree(s);
530  return str;
531 #else
532  return std::string(strerror(errno));
533 #endif
534 }
535 
536 
537 MRPT_TODO("Modify ping to run on Windows + Test this");
539  const std::string& address,
540  const int& max_attempts,
541  std::string* output_str/*=NULL*/)
542 {
543  using namespace std;
544 
545  // Format a command string
546  string cmd_str = "ping";
547 
548  // different "count" argument for Windows and *NIX systems
549 #if defined(MRPT_OS_LINUX) || defined(MRPT_OS_APPLE)
550  string count_arg_str = "-c " + num2str(max_attempts);
551 #else
552  THROW_EXCEPTION("Ping not implemented on Windows Yet.")
553 
554  string count_arg_str = "-n " + num2str(max_attempts);
555 #endif
556 
557  string redirection_str = "2>&1";
558 
559  // Join the command subparts
560  string cmd_full_str = mrpt::format("%s %s %s %s",
561  cmd_str.c_str(),
562  count_arg_str.c_str(),
563  address.c_str(),
564  redirection_str.c_str());
565 
566  // Finally exec the command
567  int code = executeCommand(cmd_full_str, output_str);
568 
569  return (code == 0);
570 }
571 
572 
573 
synch::CSemaphore sem_caller_quitted
Definition: net_utils.cpp:416
void BASE_IMPEXP memcpy(void *dest, size_t destSize, const void *src, size_t copyCount) MRPT_NO_THROWS
An OS and compiler independent version of "memcpy".
Definition: os.cpp:358
ERRORCODE_HTTP
Possible returns from a HTTP request.
Definition: net_utils.h:31
Classes for serialization, sockets, ini-file manipulation, streams, list of properties-values, timewatch, extensions to STL.
Definition: zip.h:16
For usage when passing a dynamic number of (numeric) arguments to a function, by name.
Definition: TParameters.h:51
std::vector< uint8_t > vector_byte
Definition: types_simple.h:26
This namespace provides a OS-independent interface to many useful functions: filenames manipulation...
Definition: math_frwds.h:29
#define THROW_EXCEPTION(msg)
void BASE_IMPEXP encodeBase64(const vector_byte &inputData, std::string &outString)
Encode a sequence of bytes as a string in base-64.
Definition: base64.cpp:27
STL namespace.
const Scalar * const_iterator
Definition: eigen_plugins.h:24
void sendString(const std::string &str)
Writes a string to the socket.
GLdouble s
Definition: glext.h:3602
GLenum GLsizei len
Definition: glext.h:4349
void Tic()
Starts the stopwatch.
Definition: CTicTac.cpp:77
TThreadHandle createThreadRef(void(*func)(T &), T &param)
Definition: threads.h:185
size_t readAsync(void *Buffer, const size_t Count, const int timeoutStart_ms=-1, const int timeoutBetween_ms=-1)
A method for reading from the socket with an optional timeout.
ERRORCODE_HTTP BASE_IMPEXP http_get(const string &url, vector_byte &out_content, string &out_errormsg, int port=80, const string &auth_user=string(), const string &auth_pass=string(), int *out_http_responsecode=NULL, mrpt::utils::TParameters< string > *extra_headers=NULL, mrpt::utils::TParameters< string > *out_headers=NULL, int timeout_ms=1000)
Perform an HTTP GET operation (version for retrieving the data as a vector_byte)
Definition: net_utils.cpp:388
int BASE_IMPEXP executeCommand(const std::string &command, std::string *output=NULL, const std::string &mode="r")
Execute Generic Shell Command.
Definition: os.cpp:599
MRPT_TODO("Modify ping to run on Windows + Test this")
GLuint index
Definition: glext.h:3891
void BASE_IMPEXP sleep(int time_ms) MRPT_NO_THROWS
An OS-independent method for sending the current thread to "sleep" for a given period of time...
Definition: threads.cpp:57
bool isConnected()
Returns true if this objects represents a successfully connected socket.
std::string BASE_IMPEXP getLastSocketErrorStr()
Returns a description of the last Sockets error.
Definition: net_utils.cpp:519
This class implements a high-performance stopwatch.
Definition: CTicTac.h:24
A set of useful routines for networking.
Definition: net_utils.h:23
int val
Definition: mrpt_jpeglib.h:953
std::string BASE_IMPEXP format(const char *fmt,...) MRPT_printf_format_check(1
A std::string version of C sprintf.
Definition: format.cpp:21
This namespace provides multitask, synchronization utilities.
Definition: atomic_incr.h:29
std::string BASE_IMPEXP format(const char *fmt,...) MRPT_printf_format_check(1
A std::string version of C sprintf.
GLsizei const GLchar ** string
Definition: glext.h:3919
synch::CSemaphore sem_solved
Definition: net_utils.cpp:415
const GLdouble * v
Definition: glext.h:3603
This is the global namespace for all Mobile Robot Programming Toolkit (MRPT) libraries.
std::string in_servername
Definition: net_utils.cpp:417
void connect(const std::string &remotePartAddress, unsigned short remotePartTCPPort, unsigned int timeout_ms=0)
Establishes a connection with a remote part.
Definition: inftrees.h:28
CThreadSafeVariable< std::string > out_solved_ip
Definition: net_utils.cpp:418
This structure contains the information needed to interface the threads API on each platform: ...
Definition: threads.h:25
bool BASE_IMPEXP Ping(const std::string &address, const int &max_attempts, std::string *output_str=NULL)
Ping an IP address.
Definition: net_utils.cpp:538
bool BASE_IMPEXP DNS_resolve_async(const std::string &server_name, std::string &out_ip, const unsigned int timeout_ms=3000)
Resolve a server address by its name, returning its IP address as a string - This method has a timeou...
Definition: net_utils.cpp:430
std::string num2str(T const &value)
Convert number instance to string.
Definition: string_utils.h:140
double Tac()
Stops the stopwatch.
Definition: CTicTac.cpp:92
ERRORCODE_HTTP BASE_IMPEXP http_request(const string &http_method, const string &http_send_content, const string &url, vector_byte &out_content, string &out_errormsg, int port=80, const string &auth_user=string(), const string &auth_pass=string(), int *out_http_responsecode=NULL, mrpt::utils::TParameters< string > *extra_headers=NULL, mrpt::utils::TParameters< string > *out_headers=NULL, int timeout_ms=1000)
Generic function for HTTP GET & POST methods.
Definition: net_utils.cpp:78
GLuint address
Definition: glext.h:6078
A TCP socket that can be connected to a TCP server, implementing MRPT&#39;s CStream interface for passing...
GLsizeiptr size
Definition: glext.h:3779
GLuint res
Definition: glext.h:6298
GLfloat param
Definition: glext.h:3705
GLsizei GLsizei GLenum GLenum const GLvoid * data
Definition: glext.h:3520
GLfloat GLfloat p
Definition: glext.h:5587
void BASE_IMPEXP tokenize(const std::string &inString, const std::string &inDelimiters, std::deque< std::string > &outTokens, bool skipBlankTokens=true) MRPT_NO_THROWS
Tokenizes a string according to a set of delimiting characters.
A semaphore for inter-thread synchronization.
Definition: CSemaphore.h:28
void thread_DNS_solver_async(TDNSThreadData &dat)
Definition: net_utils.cpp:472
bool has(const std::string &s) const
Definition: TParameters.h:76
void BASE_IMPEXP joinThread(const TThreadHandle &threadHandle)
Waits until the given thread ends.
Definition: threads.cpp:190



Page generated by Doxygen 1.8.14 for MRPT 1.5.6 Git: 4c65e8431 Tue Apr 24 08:18:17 2018 +0200 at lun oct 28 01:35:26 CET 2019