transfer.c@ 104083

Last change on this file since 104083 was 104083, checked in by vboxsync, 9 months ago
curl-8.7.1: Applied and adjusted our curl changes to 8.4.0. bugref:10639
Property svn:eol-style set to `native`
File size: 39.7 KB

Line
1	/***************************************************************************
2	* _ _ ____ _
3	* Project ___\| \| \| \| _ \\| \|
4	* / __\| \| \| \| \|_) \| \|
5	* \| (__\| \|_\| \| _ <\| \|___
6	* \___\|\___/\|_\| \_\_____\|
7	*
8	* Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9	*
10	* This software is licensed as described in the file COPYING, which
11	* you should have received as part of this distribution. The terms
12	* are also available at https://curl.se/docs/copyright.html.
13	*
14	* You may opt to use, copy, modify, merge, publish, distribute and/or sell
15	* copies of the Software, and permit persons to whom the Software is
16	* furnished to do so, under the terms of the COPYING file.
17	*
18	* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19	* KIND, either express or implied.
20	*
21	* SPDX-License-Identifier: curl
22	*
23	***************************************************************************/
24
25	#include "curl_setup.h"
26	#include "strtoofft.h"
27
28	#ifdef HAVE_NETINET_IN_H
29	#include <netinet/in.h>
30	#endif
31	#ifdef HAVE_NETDB_H
32	#include <netdb.h>
33	#endif
34	#ifdef HAVE_ARPA_INET_H
35	#include <arpa/inet.h>
36	#endif
37	#ifdef HAVE_NET_IF_H
38	#include <net/if.h>
39	#endif
40	#ifdef HAVE_SYS_IOCTL_H
41	#include <sys/ioctl.h>
42	#endif
43	#include <signal.h>
44
45	#ifdef HAVE_SYS_PARAM_H
46	#include <sys/param.h>
47	#endif
48
49	#ifdef HAVE_SYS_SELECT_H
50	#include <sys/select.h>
51	#elif defined(HAVE_UNISTD_H)
52	#include <unistd.h>
53	#endif
54
55	#ifndef HAVE_SOCKET
56	#error "We can't compile without socket() support!"
57	#endif
58
59	#include "urldata.h"
60	#include <curl/curl.h>
61	#include "netrc.h"
62
63	#include "content_encoding.h"
64	#include "hostip.h"
65	#include "cfilters.h"
66	#include "cw-out.h"
67	#include "transfer.h"
68	#include "sendf.h"
69	#include "speedcheck.h"
70	#include "progress.h"
71	#include "http.h"
72	#include "url.h"
73	#include "getinfo.h"
74	#include "vtls/vtls.h"
75	#include "vquic/vquic.h"
76	#include "select.h"
77	#include "multiif.h"
78	#include "connect.h"
79	#include "http2.h"
80	#include "mime.h"
81	#include "strcase.h"
82	#include "urlapi-int.h"
83	#include "hsts.h"
84	#include "setopt.h"
85	#include "headers.h"
86
87	/* The last 3 #include files should be in this order */
88	#include "curl_printf.h"
89	#include "curl_memory.h"
90	#include "memdebug.h"
91
92	#if !defined(CURL_DISABLE_HTTP) \|\| !defined(CURL_DISABLE_SMTP) \|\| \
93	!defined(CURL_DISABLE_IMAP)
94	/*
95	* checkheaders() checks the linked list of custom headers for a
96	* particular header (prefix). Provide the prefix without colon!
97	*
98	* Returns a pointer to the first matching header or NULL if none matched.
99	*/
100	char Curl_checkheaders(const struct Curl_easy data,
101	const char *thisheader,
102	const size_t thislen)
103	{
104	struct curl_slist *head;
105	DEBUGASSERT(thislen);
106	DEBUGASSERT(thisheader[thislen-1] != ':');
107
108	for(head = data->set.headers; head; head = head->next) {
109	if(strncasecompare(head->data, thisheader, thislen) &&
110	Curl_headersep(head->data[thislen]) )
111	return head->data;
112	}
113
114	return NULL;
115	}
116	#endif
117
118	static int data_pending(struct Curl_easy *data)
119	{
120	struct connectdata *conn = data->conn;
121
122	if(conn->handler->protocol&PROTO_FAMILY_FTP)
123	return Curl_conn_data_pending(data, SECONDARYSOCKET);
124
125	/* in the case of libssh2, we can never be really sure that we have emptied
126	its internal buffers so we MUST always try until we get EAGAIN back */
127	return conn->handler->protocol&(CURLPROTO_SCP\|CURLPROTO_SFTP) \|\|
128	Curl_conn_data_pending(data, FIRSTSOCKET);
129	}
130
131	/*
132	* Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the
133	* remote document with the time provided by CURLOPT_TIMEVAL
134	*/
135	bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc)
136	{
137	if((timeofdoc == 0) \|\| (data->set.timevalue == 0))
138	return TRUE;
139
140	switch(data->set.timecondition) {
141	case CURL_TIMECOND_IFMODSINCE:
142	default:
143	if(timeofdoc <= data->set.timevalue) {
144	infof(data,
145	"The requested document is not new enough");
146	data->info.timecond = TRUE;
147	return FALSE;
148	}
149	break;
150	case CURL_TIMECOND_IFUNMODSINCE:
151	if(timeofdoc >= data->set.timevalue) {
152	infof(data,
153	"The requested document is not old enough");
154	data->info.timecond = TRUE;
155	return FALSE;
156	}
157	break;
158	}
159
160	return TRUE;
161	}
162
163	/**
164	* Receive raw response data for the transfer.
165	* @param data the transfer
166	* @param buf buffer to keep response data received
167	* @param blen length of `buf`
168	* @param eos_reliable if EOS detection in underlying connection is reliable
169	* @param err error code in case of -1 return
170	* @return number of bytes read or -1 for error
171	*/
172	static ssize_t Curl_xfer_recv_resp(struct Curl_easy *data,
173	char *buf, size_t blen,
174	bool eos_reliable,
175	CURLcode *err)
176	{
177	ssize_t nread;
178
179	DEBUGASSERT(blen > 0);
180	/* If we are reading BODY data and the connection does NOT handle EOF
181	* and we know the size of the BODY data, limit the read amount */
182	if(!eos_reliable && !data->req.header && data->req.size != -1) {
183	curl_off_t totalleft = data->req.size - data->req.bytecount;
184	if(totalleft <= 0)
185	blen = 0;
186	else if(totalleft < (curl_off_t)blen)
187	blen = (size_t)totalleft;
188	}
189
190	if(!blen) {
191	/* want nothing - continue as if read nothing. */
192	DEBUGF(infof(data, "readwrite_data: we're done"));
193	*err = CURLE_OK;
194	return 0;
195	}
196
197	*err = Curl_xfer_recv(data, buf, blen, &nread);
198	if(*err)
199	return -1;
200	DEBUGASSERT(nread >= 0);
201	*err = CURLE_OK;
202	return nread;
203	}
204
205	/*
206	* Go ahead and do a read if we have a readable socket or if
207	* the stream was rewound (in which case we have data in a
208	* buffer)
209	*/
210	static CURLcode readwrite_data(struct Curl_easy *data,
211	struct SingleRequest *k,
212	int *didwhat)
213	{
214	struct connectdata *conn = data->conn;
215	CURLcode result = CURLE_OK;
216	char buf, xfer_buf;
217	size_t blen, xfer_blen;
218	int maxloops = 10;
219	curl_off_t total_received = 0;
220	bool is_multiplex = FALSE;
221
222	result = Curl_multi_xfer_buf_borrow(data, &xfer_buf, &xfer_blen);
223	if(result)
224	goto out;
225
226	/* This is where we loop until we have read everything there is to
227	read or we get a CURLE_AGAIN */
228	do {
229	bool is_eos = FALSE;
230	size_t bytestoread;
231	ssize_t nread;
232
233	if(!is_multiplex) {
234	/* Multiplexed connection have inherent handling of EOF and we do not
235	* have to carefully restrict the amount we try to read.
236	* Multiplexed changes only in one direction. */
237	is_multiplex = Curl_conn_is_multiplex(conn, FIRSTSOCKET);
238	}
239
240	buf = xfer_buf;
241	bytestoread = xfer_blen;
242
243	if(bytestoread && data->set.max_recv_speed) {
244	/* In case of speed limit on receiving: if this loop already got
245	* data, break out. If not, limit the amount of bytes to receive.
246	* The overall, timed, speed limiting is done in multi.c */
247	if(total_received)
248	break;
249	if((size_t)data->set.max_recv_speed < bytestoread)
250	bytestoread = (size_t)data->set.max_recv_speed;
251	}
252
253	nread = Curl_xfer_recv_resp(data, buf, bytestoread,
254	is_multiplex, &result);
255	if(nread < 0) {
256	if(CURLE_AGAIN == result) {
257	result = CURLE_OK;
258	break; /* get out of loop */
259	}
260	goto out; /* real error */
261	}
262
263	/* We only get a 0-length read on EndOfStream */
264	blen = (size_t)nread;
265	is_eos = (blen == 0);
266	*didwhat \|= KEEP_RECV;
267
268	if(!blen) {
269	/* if we receive 0 or less here, either the data transfer is done or the
270	server closed the connection and we bail out from this! */
271	if(is_multiplex)
272	DEBUGF(infof(data, "nread == 0, stream closed, bailing"));
273	else
274	DEBUGF(infof(data, "nread <= 0, server closed connection, bailing"));
275	if(k->eos_written) { /* already did write this to client, leave */
276	k->keepon = 0; /* stop sending as well */
277	break;
278	}
279	}
280	total_received += blen;
281
282	result = Curl_xfer_write_resp(data, buf, blen, is_eos);
283	if(result \|\| data->req.done)
284	goto out;
285
286	/* if we are done, we stop receiving. On multiplexed connections,
287	* we should read the EOS. Which may arrive as meta data after
288	* the bytes. Not taking it in might lead to RST of streams. */
289	if((!is_multiplex && data->req.download_done) \|\| is_eos) {
290	data->req.keepon &= ~KEEP_RECV;
291	}
292	/* if we are PAUSEd or stopped receiving, leave the loop */
293	if((k->keepon & KEEP_RECV_PAUSE) \|\| !(k->keepon & KEEP_RECV))
294	break;
295
296	} while(maxloops-- && data_pending(data));
297
298	if(maxloops <= 0) {
299	/* did not read until EAGAIN, mark read-again-please */
300	data->state.select_bits = CURL_CSELECT_IN;
301	if((k->keepon & KEEP_SENDBITS) == KEEP_SEND)
302	data->state.select_bits \|= CURL_CSELECT_OUT;
303	}
304
305	if(((k->keepon & (KEEP_RECV\|KEEP_SEND)) == KEEP_SEND) &&
306	(conn->bits.close \|\| is_multiplex)) {
307	/* When we've read the entire thing and the close bit is set, the server
308	may now close the connection. If there's now any kind of sending going
309	on from our side, we need to stop that immediately. */
310	infof(data, "we are done reading and this is set to close, stop send");
311	k->keepon &= ~KEEP_SEND; /* no writing anymore either */
312	k->keepon &= ~KEEP_SEND_PAUSE; /* no pausing anymore either */
313	}
314
315	out:
316	Curl_multi_xfer_buf_release(data, xfer_buf);
317	if(result)
318	DEBUGF(infof(data, "readwrite_data() -> %d", result));
319	return result;
320	}
321
322	#if defined(_WIN32) && defined(USE_WINSOCK)
323	#ifndef SIO_IDEAL_SEND_BACKLOG_QUERY
324	#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747B
325	#endif
326
327	static void win_update_buffer_size(curl_socket_t sockfd)
328	{
329	int result;
330	ULONG ideal;
331	DWORD ideallen;
332	result = WSAIoctl(sockfd, SIO_IDEAL_SEND_BACKLOG_QUERY, 0, 0,
333	&ideal, sizeof(ideal), &ideallen, 0, 0);
334	if(result == 0) {
335	setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF,
336	(const char *)&ideal, sizeof(ideal));
337	}
338	}
339	#else
340	#define win_update_buffer_size(x)
341	#endif
342
343	#define curl_upload_refill_watermark(data) \
344	((size_t)((data)->set.upload_buffer_size >> 5))
345
346	/*
347	* Send data to upload to the server, when the socket is writable.
348	*/
349	static CURLcode readwrite_upload(struct Curl_easy data, int didwhat)
350	{
351	CURLcode result = CURLE_OK;
352
353	if((data->req.keepon & KEEP_SEND_PAUSE))
354	return CURLE_OK;
355
356	/* We should not get here when the sending is already done. It
357	* probably means that someone set `data-req.keepon \|= KEEP_SEND`
358	* when it should not. */
359	DEBUGASSERT(!Curl_req_done_sending(data));
360
361	if(!Curl_req_done_sending(data)) {
362	*didwhat \|= KEEP_SEND;
363	result = Curl_req_send_more(data);
364	if(result)
365	return result;
366
367	#if defined(_WIN32) && defined(USE_WINSOCK)
368	/* FIXME: this looks like it would fit better into cf-socket.c
369	* but then I do not know enough Windows to say... */
370	{
371	struct curltime n = Curl_now();
372	if(Curl_timediff(n, data->conn->last_sndbuf_update) > 1000) {
373	win_update_buffer_size(data->conn->writesockfd);
374	data->conn->last_sndbuf_update = n;
375	}
376	}
377	#endif
378	}
379	return result;
380	}
381
382	static int select_bits_paused(struct Curl_easy *data, int select_bits)
383	{
384	/* See issue #11982: we really need to be careful not to progress
385	* a transfer direction when that direction is paused. Not all parts
386	* of our state machine are handling PAUSED transfers correctly. So, we
387	* do not want to go there.
388	* NOTE: we are only interested in PAUSE, not HOLD. */
389
390	/* if there is data in a direction not paused, return false */
391	if(((select_bits & CURL_CSELECT_IN) &&
392	!(data->req.keepon & KEEP_RECV_PAUSE)) \|\|
393	((select_bits & CURL_CSELECT_OUT) &&
394	!(data->req.keepon & KEEP_SEND_PAUSE)))
395	return FALSE;
396
397	return (data->req.keepon & (KEEP_RECV_PAUSE\|KEEP_SEND_PAUSE));
398	}
399
400	/*
401	* Curl_readwrite() is the low-level function to be called when data is to
402	* be read and written to/from the connection.
403	*/
404	CURLcode Curl_readwrite(struct Curl_easy *data)
405	{
406	struct connectdata *conn = data->conn;
407	struct SingleRequest *k = &data->req;
408	CURLcode result;
409	struct curltime now;
410	int didwhat = 0;
411	int select_bits;
412
413	if(data->state.select_bits) {
414	if(select_bits_paused(data, data->state.select_bits)) {
415	/* leave the bits unchanged, so they'll tell us what to do when
416	* this transfer gets unpaused. */
417	DEBUGF(infof(data, "readwrite, select_bits, early return on PAUSED"));
418	result = CURLE_OK;
419	goto out;
420	}
421	select_bits = data->state.select_bits;
422	data->state.select_bits = 0;
423	}
424	else {
425	curl_socket_t fd_read;
426	curl_socket_t fd_write;
427	/* only use the proper socket if the *_HOLD bit is not set simultaneously
428	as then we are in rate limiting state in that transfer direction */
429	if((k->keepon & KEEP_RECVBITS) == KEEP_RECV)
430	fd_read = conn->sockfd;
431	else
432	fd_read = CURL_SOCKET_BAD;
433
434	if((k->keepon & KEEP_SENDBITS) == KEEP_SEND)
435	fd_write = conn->writesockfd;
436	else
437	fd_write = CURL_SOCKET_BAD;
438
439	select_bits = Curl_socket_check(fd_read, CURL_SOCKET_BAD, fd_write, 0);
440	}
441
442	if(select_bits == CURL_CSELECT_ERR) {
443	failf(data, "select/poll returned error");
444	result = CURLE_SEND_ERROR;
445	goto out;
446	}
447
448	#ifdef USE_HYPER
449	if(conn->datastream) {
450	result = conn->datastream(data, conn, &didwhat, select_bits);
451	if(result \|\| data->req.done)
452	goto out;
453	}
454	else {
455	#endif
456	/* We go ahead and do a read if we have a readable socket or if
457	the stream was rewound (in which case we have data in a
458	buffer) */
459	if((k->keepon & KEEP_RECV) && (select_bits & CURL_CSELECT_IN)) {
460	result = readwrite_data(data, k, &didwhat);
461	if(result \|\| data->req.done)
462	goto out;
463	}
464
465	/* If we still have writing to do, we check if we have a writable socket. */
466	if(((k->keepon & KEEP_SEND) && (select_bits & CURL_CSELECT_OUT)) \|\|
467	(k->keepon & KEEP_SEND_TIMED)) {
468	/* write */
469
470	result = readwrite_upload(data, &didwhat);
471	if(result)
472	goto out;
473	}
474	#ifdef USE_HYPER
475	}
476	#endif
477
478	now = Curl_now();
479	if(!didwhat) {
480	result = Curl_conn_ev_data_idle(data);
481	if(result)
482	goto out;
483	}
484
485	if(Curl_pgrsUpdate(data))
486	result = CURLE_ABORTED_BY_CALLBACK;
487	else
488	result = Curl_speedcheck(data, now);
489	if(result)
490	goto out;
491
492	if(k->keepon) {
493	if(0 > Curl_timeleft(data, &now, FALSE)) {
494	if(k->size != -1) {
495	failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
496	" milliseconds with %" CURL_FORMAT_CURL_OFF_T " out of %"
497	CURL_FORMAT_CURL_OFF_T " bytes received",
498	Curl_timediff(now, data->progress.t_startsingle),
499	k->bytecount, k->size);
500	}
501	else {
502	failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
503	" milliseconds with %" CURL_FORMAT_CURL_OFF_T " bytes received",
504	Curl_timediff(now, data->progress.t_startsingle),
505	k->bytecount);
506	}
507	result = CURLE_OPERATION_TIMEDOUT;
508	goto out;
509	}
510	}
511	else {
512	/*
513	* The transfer has been performed. Just make some general checks before
514	* returning.
515	*/
516	if(!(data->req.no_body) && (k->size != -1) &&
517	(k->bytecount != k->size) &&
518	#ifdef CURL_DO_LINEEND_CONV
519	/* Most FTP servers don't adjust their file SIZE response for CRLFs,
520	so we'll check to see if the discrepancy can be explained
521	by the number of CRLFs we've changed to LFs.
522	*/
523	(k->bytecount != (k->size + data->state.crlf_conversions)) &&
524	#endif /* CURL_DO_LINEEND_CONV */
525	!k->newurl) {
526	failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T
527	" bytes remaining to read", k->size - k->bytecount);
528	result = CURLE_PARTIAL_FILE;
529	goto out;
530	}
531	if(Curl_pgrsUpdate(data)) {
532	result = CURLE_ABORTED_BY_CALLBACK;
533	goto out;
534	}
535	}
536
537	/* If there is nothing more to send/recv, the request is done */
538	if(0 == (k->keepon&(KEEP_RECVBITS\|KEEP_SENDBITS)))
539	data->req.done = TRUE;
540
541	out:
542	if(result)
543	DEBUGF(infof(data, "Curl_readwrite() -> %d", result));
544	return result;
545	}
546
547	/* Curl_init_CONNECT() gets called each time the handle switches to CONNECT
548	which means this gets called once for each subsequent redirect etc */
549	void Curl_init_CONNECT(struct Curl_easy *data)
550	{
551	data->state.fread_func = data->set.fread_func_set;
552	data->state.in = data->set.in_set;
553	data->state.upload = (data->state.httpreq == HTTPREQ_PUT);
554	}
555
556	/*
557	* Curl_pretransfer() is called immediately before a transfer starts, and only
558	* once for one transfer no matter if it has redirects or do multi-pass
559	* authentication etc.
560	*/
561	CURLcode Curl_pretransfer(struct Curl_easy *data)
562	{
563	CURLcode result;
564
565	if(!data->state.url && !data->set.uh) {
566	/* we can't do anything without URL */
567	failf(data, "No URL set");
568	return CURLE_URL_MALFORMAT;
569	}
570
571	/* since the URL may have been redirected in a previous use of this handle */
572	if(data->state.url_alloc) {
573	/* the already set URL is allocated, free it first! */
574	Curl_safefree(data->state.url);
575	data->state.url_alloc = FALSE;
576	}
577
578	if(!data->state.url && data->set.uh) {
579	CURLUcode uc;
580	free(data->set.str[STRING_SET_URL]);
581	uc = curl_url_get(data->set.uh,
582	CURLUPART_URL, &data->set.str[STRING_SET_URL], 0);
583	if(uc) {
584	failf(data, "No URL set");
585	return CURLE_URL_MALFORMAT;
586	}
587	}
588
589	if(data->set.postfields && data->set.set_resume_from) {
590	/* we can't */
591	failf(data, "cannot mix POSTFIELDS with RESUME_FROM");
592	return CURLE_BAD_FUNCTION_ARGUMENT;
593	}
594
595	data->state.prefer_ascii = data->set.prefer_ascii;
596	#ifdef CURL_LIST_ONLY_PROTOCOL
597	data->state.list_only = data->set.list_only;
598	#endif
599	data->state.httpreq = data->set.method;
600	data->state.url = data->set.str[STRING_SET_URL];
601
602	/* Init the SSL session ID cache here. We do it here since we want to do it
603	after the *_setopt() calls (that could specify the size of the cache) but
604	before any transfer takes place. */
605	result = Curl_ssl_initsessions(data, data->set.general_ssl.max_ssl_sessions);
606	if(result)
607	return result;
608
609	data->state.requests = 0;
610	data->state.followlocation = 0; /* reset the location-follow counter */
611	data->state.this_is_a_follow = FALSE; /* reset this */
612	data->state.errorbuf = FALSE; /* no error has occurred */
613	data->state.httpwant = data->set.httpwant;
614	data->state.httpversion = 0;
615	data->state.authproblem = FALSE;
616	data->state.authhost.want = data->set.httpauth;
617	data->state.authproxy.want = data->set.proxyauth;
618	Curl_safefree(data->info.wouldredirect);
619	Curl_data_priority_clear_state(data);
620
621	if(data->state.httpreq == HTTPREQ_PUT)
622	data->state.infilesize = data->set.filesize;
623	else if((data->state.httpreq != HTTPREQ_GET) &&
624	(data->state.httpreq != HTTPREQ_HEAD)) {
625	data->state.infilesize = data->set.postfieldsize;
626	if(data->set.postfields && (data->state.infilesize == -1))
627	data->state.infilesize = (curl_off_t)strlen(data->set.postfields);
628	}
629	else
630	data->state.infilesize = 0;
631
632	/* If there is a list of cookie files to read, do it now! */
633	Curl_cookie_loadfiles(data);
634
635	/* If there is a list of host pairs to deal with */
636	if(data->state.resolve)
637	result = Curl_loadhostpairs(data);
638
639	/* If there is a list of hsts files to read */
640	Curl_hsts_loadfiles(data);
641
642	if(!result) {
643	/* Allow data->set.use_port to set which port to use. This needs to be
644	* disabled for example when we follow Location: headers to URLs using
645	* different ports! */
646	data->state.allow_port = TRUE;
647
648	#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
649	/*************************************************************
650	* Tell signal handler to ignore SIGPIPE
651	*************************************************************/
652	if(!data->set.no_signal)
653	data->state.prev_signal = signal(SIGPIPE, SIG_IGN);
654	#endif
655
656	Curl_initinfo(data); /* reset session-specific information "variables" */
657	Curl_pgrsResetTransferSizes(data);
658	Curl_pgrsStartNow(data);
659
660	/* In case the handle is reused and an authentication method was picked
661	in the session we need to make sure we only use the one(s) we now
662	consider to be fine */
663	data->state.authhost.picked &= data->state.authhost.want;
664	data->state.authproxy.picked &= data->state.authproxy.want;
665
666	#ifndef CURL_DISABLE_FTP
667	data->state.wildcardmatch = data->set.wildcard_enabled;
668	if(data->state.wildcardmatch) {
669	struct WildcardData *wc;
670	if(!data->wildcard) {
671	data->wildcard = calloc(1, sizeof(struct WildcardData));
672	if(!data->wildcard)
673	return CURLE_OUT_OF_MEMORY;
674	}
675	wc = data->wildcard;
676	if(wc->state < CURLWC_INIT) {
677	if(wc->ftpwc)
678	wc->dtor(wc->ftpwc);
679	Curl_safefree(wc->pattern);
680	Curl_safefree(wc->path);
681	result = Curl_wildcard_init(wc); /* init wildcard structures */
682	if(result)
683	return CURLE_OUT_OF_MEMORY;
684	}
685	}
686	#endif
687	result = Curl_hsts_loadcb(data, data->hsts);
688	}
689
690	/*
691	* Set user-agent. Used for HTTP, but since we can attempt to tunnel
692	* basically anything through an HTTP proxy we can't limit this based on
693	* protocol.
694	*/
695	if(data->set.str[STRING_USERAGENT]) {
696	Curl_safefree(data->state.aptr.uagent);
697	data->state.aptr.uagent =
698	aprintf("User-Agent: %s\r\n", data->set.str[STRING_USERAGENT]);
699	if(!data->state.aptr.uagent)
700	return CURLE_OUT_OF_MEMORY;
701	}
702
703	if(!result)
704	result = Curl_setstropt(&data->state.aptr.user,
705	data->set.str[STRING_USERNAME]);
706	if(!result)
707	result = Curl_setstropt(&data->state.aptr.passwd,
708	data->set.str[STRING_PASSWORD]);
709	if(!result)
710	result = Curl_setstropt(&data->state.aptr.proxyuser,
711	data->set.str[STRING_PROXYUSERNAME]);
712	if(!result)
713	result = Curl_setstropt(&data->state.aptr.proxypasswd,
714	data->set.str[STRING_PROXYPASSWORD]);
715
716	data->req.headerbytecount = 0;
717	Curl_headers_cleanup(data);
718	return result;
719	}
720
721	/*
722	* Curl_posttransfer() is called immediately after a transfer ends
723	*/
724	CURLcode Curl_posttransfer(struct Curl_easy *data)
725	{
726	#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
727	/* restore the signal handler for SIGPIPE before we get back */
728	if(!data->set.no_signal)
729	signal(SIGPIPE, data->state.prev_signal);
730	#else
731	(void)data; /* unused parameter */
732	#endif
733
734	return CURLE_OK;
735	}
736
737	/*
738	* Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string
739	* as given by the remote server and set up the new URL to request.
740	*
741	* This function DOES NOT FREE the given url.
742	*/
743	CURLcode Curl_follow(struct Curl_easy *data,
744	char newurl, / the Location: string */
745	followtype type) /* see transfer.h */
746	{
747	#ifdef CURL_DISABLE_HTTP
748	(void)data;
749	(void)newurl;
750	(void)type;
751	/* Location: following will not happen when HTTP is disabled */
752	return CURLE_TOO_MANY_REDIRECTS;
753	#else
754
755	/* Location: redirect */
756	bool disallowport = FALSE;
757	bool reachedmax = FALSE;
758	CURLUcode uc;
759
760	DEBUGASSERT(type != FOLLOW_NONE);
761
762	if(type != FOLLOW_FAKE)
763	data->state.requests++; /* count all real follows */
764	if(type == FOLLOW_REDIR) {
765	if((data->set.maxredirs != -1) &&
766	(data->state.followlocation >= data->set.maxredirs)) {
767	reachedmax = TRUE;
768	type = FOLLOW_FAKE; /* switch to fake to store the would-be-redirected
769	to URL */
770	}
771	else {
772	data->state.followlocation++; /* count redirect-followings, including
773	auth reloads */
774
775	if(data->set.http_auto_referer) {
776	CURLU *u;
777	char *referer = NULL;
778
779	/* We are asked to automatically set the previous URL as the referer
780	when we get the next URL. We pick the ->url field, which may or may
781	not be 100% correct */
782
783	if(data->state.referer_alloc) {
784	Curl_safefree(data->state.referer);
785	data->state.referer_alloc = FALSE;
786	}
787
788	/* Make a copy of the URL without credentials and fragment */
789	u = curl_url();
790	if(!u)
791	return CURLE_OUT_OF_MEMORY;
792
793	uc = curl_url_set(u, CURLUPART_URL, data->state.url, 0);
794	if(!uc)
795	uc = curl_url_set(u, CURLUPART_FRAGMENT, NULL, 0);
796	if(!uc)
797	uc = curl_url_set(u, CURLUPART_USER, NULL, 0);
798	if(!uc)
799	uc = curl_url_set(u, CURLUPART_PASSWORD, NULL, 0);
800	if(!uc)
801	uc = curl_url_get(u, CURLUPART_URL, &referer, 0);
802
803	curl_url_cleanup(u);
804
805	if(uc \|\| !referer)
806	return CURLE_OUT_OF_MEMORY;
807
808	data->state.referer = referer;
809	data->state.referer_alloc = TRUE; /* yes, free this later */
810	}
811	}
812	}
813
814	if((type != FOLLOW_RETRY) &&
815	(data->req.httpcode != 401) && (data->req.httpcode != 407) &&
816	Curl_is_absolute_url(newurl, NULL, 0, FALSE)) {
817	/* If this is not redirect due to a 401 or 407 response and an absolute
818	URL: don't allow a custom port number */
819	disallowport = TRUE;
820	}
821
822	DEBUGASSERT(data->state.uh);
823	uc = curl_url_set(data->state.uh, CURLUPART_URL, newurl,
824	(type == FOLLOW_FAKE) ? CURLU_NON_SUPPORT_SCHEME :
825	((type == FOLLOW_REDIR) ? CURLU_URLENCODE : 0) \|
826	CURLU_ALLOW_SPACE \|
827	(data->set.path_as_is ? CURLU_PATH_AS_IS : 0));
828	if(uc) {
829	if(type != FOLLOW_FAKE) {
830	failf(data, "The redirect target URL could not be parsed: %s",
831	curl_url_strerror(uc));
832	return Curl_uc_to_curlcode(uc);
833	}
834
835	/* the URL could not be parsed for some reason, but since this is FAKE
836	mode, just duplicate the field as-is */
837	newurl = strdup(newurl);
838	if(!newurl)
839	return CURLE_OUT_OF_MEMORY;
840	}
841	else {
842	uc = curl_url_get(data->state.uh, CURLUPART_URL, &newurl, 0);
843	if(uc)
844	return Curl_uc_to_curlcode(uc);
845
846	/* Clear auth if this redirects to a different port number or protocol,
847	unless permitted */
848	if(!data->set.allow_auth_to_other_hosts && (type != FOLLOW_FAKE)) {
849	char *portnum;
850	int port;
851	bool clear = FALSE;
852
853	if(data->set.use_port && data->state.allow_port)
854	/* a custom port is used */
855	port = (int)data->set.use_port;
856	else {
857	uc = curl_url_get(data->state.uh, CURLUPART_PORT, &portnum,
858	CURLU_DEFAULT_PORT);
859	if(uc) {
860	free(newurl);
861	return Curl_uc_to_curlcode(uc);
862	}
863	port = atoi(portnum);
864	free(portnum);
865	}
866	if(port != data->info.conn_remote_port) {
867	infof(data, "Clear auth, redirects to port from %u to %u",
868	data->info.conn_remote_port, port);
869	clear = TRUE;
870	}
871	else {
872	char *scheme;
873	const struct Curl_handler *p;
874	uc = curl_url_get(data->state.uh, CURLUPART_SCHEME, &scheme, 0);
875	if(uc) {
876	free(newurl);
877	return Curl_uc_to_curlcode(uc);
878	}
879
880	p = Curl_get_scheme_handler(scheme);
881	if(p && (p->protocol != data->info.conn_protocol)) {
882	infof(data, "Clear auth, redirects scheme from %s to %s",
883	data->info.conn_scheme, scheme);
884	clear = TRUE;
885	}
886	free(scheme);
887	}
888	if(clear) {
889	Curl_safefree(data->state.aptr.user);
890	Curl_safefree(data->state.aptr.passwd);
891	}
892	}
893	}
894
895	if(type == FOLLOW_FAKE) {
896	/* we're only figuring out the new url if we would've followed locations
897	but now we're done so we can get out! */
898	data->info.wouldredirect = newurl;
899
900	if(reachedmax) {
901	failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs);
902	return CURLE_TOO_MANY_REDIRECTS;
903	}
904	return CURLE_OK;
905	}
906
907	if(disallowport)
908	data->state.allow_port = FALSE;
909
910	if(data->state.url_alloc)
911	Curl_safefree(data->state.url);
912
913	data->state.url = newurl;
914	data->state.url_alloc = TRUE;
915	Curl_req_soft_reset(&data->req, data);
916	infof(data, "Issue another request to this URL: '%s'", data->state.url);
917
918	/*
919	* We get here when the HTTP code is 300-399 (and 401). We need to perform
920	* differently based on exactly what return code there was.
921	*
922	* News from 7.10.6: we can also get here on a 401 or 407, in case we act on
923	* an HTTP (proxy-) authentication scheme other than Basic.
924	*/
925	switch(data->info.httpcode) {
926	/* 401 - Act on a WWW-Authenticate, we keep on moving and do the
927	Authorization: XXXX header in the HTTP request code snippet */
928	/* 407 - Act on a Proxy-Authenticate, we keep on moving and do the
929	Proxy-Authorization: XXXX header in the HTTP request code snippet */
930	/* 300 - Multiple Choices */
931	/* 306 - Not used */
932	/* 307 - Temporary Redirect */
933	default: /* for all above (and the unknown ones) */
934	/* Some codes are explicitly mentioned since I've checked RFC2616 and they
935	* seem to be OK to POST to.
936	*/
937	break;
938	case 301: /* Moved Permanently */
939	/* (quote from RFC7231, section 6.4.2)
940	*
941	* Note: For historical reasons, a user agent MAY change the request
942	* method from POST to GET for the subsequent request. If this
943	* behavior is undesired, the 307 (Temporary Redirect) status code
944	* can be used instead.
945	*
946	* ----
947	*
948	* Many webservers expect this, so these servers often answers to a POST
949	* request with an error page. To be sure that libcurl gets the page that
950	* most user agents would get, libcurl has to force GET.
951	*
952	* This behavior is forbidden by RFC1945 and the obsolete RFC2616, and
953	* can be overridden with CURLOPT_POSTREDIR.
954	*/
955	if((data->state.httpreq == HTTPREQ_POST
956	\|\| data->state.httpreq == HTTPREQ_POST_FORM
957	\|\| data->state.httpreq == HTTPREQ_POST_MIME)
958	&& !(data->set.keep_post & CURL_REDIR_POST_301)) {
959	infof(data, "Switch from POST to GET");
960	data->state.httpreq = HTTPREQ_GET;
961	Curl_creader_set_rewind(data, FALSE);
962	}
963	break;
964	case 302: /* Found */
965	/* (quote from RFC7231, section 6.4.3)
966	*
967	* Note: For historical reasons, a user agent MAY change the request
968	* method from POST to GET for the subsequent request. If this
969	* behavior is undesired, the 307 (Temporary Redirect) status code
970	* can be used instead.
971	*
972	* ----
973	*
974	* Many webservers expect this, so these servers often answers to a POST
975	* request with an error page. To be sure that libcurl gets the page that
976	* most user agents would get, libcurl has to force GET.
977	*
978	* This behavior is forbidden by RFC1945 and the obsolete RFC2616, and
979	* can be overridden with CURLOPT_POSTREDIR.
980	*/
981	if((data->state.httpreq == HTTPREQ_POST
982	\|\| data->state.httpreq == HTTPREQ_POST_FORM
983	\|\| data->state.httpreq == HTTPREQ_POST_MIME)
984	&& !(data->set.keep_post & CURL_REDIR_POST_302)) {
985	infof(data, "Switch from POST to GET");
986	data->state.httpreq = HTTPREQ_GET;
987	Curl_creader_set_rewind(data, FALSE);
988	}
989	break;
990
991	case 303: /* See Other */
992	/* 'See Other' location is not the resource but a substitute for the
993	* resource. In this case we switch the method to GET/HEAD, unless the
994	* method is POST and the user specified to keep it as POST.
995	* https://github.com/curl/curl/issues/5237#issuecomment-614641049
996	*/
997	if(data->state.httpreq != HTTPREQ_GET &&
998	((data->state.httpreq != HTTPREQ_POST &&
999	data->state.httpreq != HTTPREQ_POST_FORM &&
1000	data->state.httpreq != HTTPREQ_POST_MIME) \|\|
1001	!(data->set.keep_post & CURL_REDIR_POST_303))) {
1002	data->state.httpreq = HTTPREQ_GET;
1003	infof(data, "Switch to %s",
1004	data->req.no_body?"HEAD":"GET");
1005	}
1006	break;
1007	case 304: /* Not Modified */
1008	/* 304 means we did a conditional request and it was "Not modified".
1009	* We shouldn't get any Location: header in this response!
1010	*/
1011	break;
1012	case 305: /* Use Proxy */
1013	/* (quote from RFC2616, section 10.3.6):
1014	* "The requested resource MUST be accessed through the proxy given
1015	* by the Location field. The Location field gives the URI of the
1016	* proxy. The recipient is expected to repeat this single request
1017	* via the proxy. 305 responses MUST only be generated by origin
1018	* servers."
1019	*/
1020	break;
1021	}
1022	Curl_pgrsTime(data, TIMER_REDIRECT);
1023	Curl_pgrsResetTransferSizes(data);
1024
1025	return CURLE_OK;
1026	#endif /* CURL_DISABLE_HTTP */
1027	}
1028
1029	/* Returns CURLE_OK and sets '*url' if a request retry is wanted.
1030
1031	NOTE: that the url is malloc()ed. /
1032	CURLcode Curl_retry_request(struct Curl_easy data, char *url)
1033	{
1034	struct connectdata *conn = data->conn;
1035	bool retry = FALSE;
1036	*url = NULL;
1037
1038	/* if we're talking upload, we can't do the checks below, unless the protocol
1039	is HTTP as when uploading over HTTP we will still get a response */
1040	if(data->state.upload &&
1041	!(conn->handler->protocol&(PROTO_FAMILY_HTTP\|CURLPROTO_RTSP)))
1042	return CURLE_OK;
1043
1044	if((data->req.bytecount + data->req.headerbytecount == 0) &&
1045	conn->bits.reuse &&
1046	(!data->req.no_body \|\| (conn->handler->protocol & PROTO_FAMILY_HTTP))
1047	#ifndef CURL_DISABLE_RTSP
1048	&& (data->set.rtspreq != RTSPREQ_RECEIVE)
1049	#endif
1050	)
1051	/* We got no data, we attempted to reuse a connection. For HTTP this
1052	can be a retry so we try again regardless if we expected a body.
1053	For other protocols we only try again only if we expected a body.
1054
1055	This might happen if the connection was left alive when we were
1056	done using it before, but that was closed when we wanted to read from
1057	it again. Bad luck. Retry the same request on a fresh connect! */
1058	retry = TRUE;
1059	else if(data->state.refused_stream &&
1060	(data->req.bytecount + data->req.headerbytecount == 0) ) {
1061	/* This was sent on a refused stream, safe to rerun. A refused stream
1062	error can typically only happen on HTTP/2 level if the stream is safe
1063	to issue again, but the nghttp2 API can deliver the message to other
1064	streams as well, which is why this adds the check the data counters
1065	too. */
1066	infof(data, "REFUSED_STREAM, retrying a fresh connect");
1067	data->state.refused_stream = FALSE; /* clear again */
1068	retry = TRUE;
1069	}
1070	if(retry) {
1071	#define CONN_MAX_RETRIES 5
1072	if(data->state.retrycount++ >= CONN_MAX_RETRIES) {
1073	failf(data, "Connection died, tried %d times before giving up",
1074	CONN_MAX_RETRIES);
1075	data->state.retrycount = 0;
1076	return CURLE_SEND_ERROR;
1077	}
1078	infof(data, "Connection died, retrying a fresh connect (retry count: %d)",
1079	data->state.retrycount);
1080	*url = strdup(data->state.url);
1081	if(!*url)
1082	return CURLE_OUT_OF_MEMORY;
1083
1084	connclose(conn, "retry"); /* close this connection */
1085	conn->bits.retry = TRUE; /* mark this as a connection we're about
1086	to retry. Marking it this way should
1087	prevent i.e HTTP transfers to return
1088	error just because nothing has been
1089	transferred! */
1090	Curl_creader_set_rewind(data, TRUE);
1091	}
1092	return CURLE_OK;
1093	}
1094
1095	/*
1096	* Curl_xfer_setup() is called to setup some basic properties for the
1097	* upcoming transfer.
1098	*/
1099	void Curl_xfer_setup(
1100	struct Curl_easy data, / transfer */
1101	int sockindex, /* socket index to read from or -1 */
1102	curl_off_t size, /* -1 if unknown at this point */
1103	bool getheader, /* TRUE if header parsing is wanted */
1104	int writesockindex /* socket index to write to, it may very well be
1105	the same we read from. -1 disables */
1106	)
1107	{
1108	struct SingleRequest *k = &data->req;
1109	struct connectdata *conn = data->conn;
1110	bool want_send = Curl_req_want_send(data);
1111
1112	DEBUGASSERT(conn != NULL);
1113	DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
1114	DEBUGASSERT((writesockindex <= 1) && (writesockindex >= -1));
1115
1116	if(conn->bits.multiplex \|\| conn->httpversion >= 20 \|\| want_send) {
1117	/* when multiplexing, the read/write sockets need to be the same! */
1118	conn->sockfd = sockindex == -1 ?
1119	((writesockindex == -1 ? CURL_SOCKET_BAD : conn->sock[writesockindex])) :
1120	conn->sock[sockindex];
1121	conn->writesockfd = conn->sockfd;
1122	if(want_send)
1123	/* special and very HTTP-specific */
1124	writesockindex = FIRSTSOCKET;
1125	}
1126	else {
1127	conn->sockfd = sockindex == -1 ?
1128	CURL_SOCKET_BAD : conn->sock[sockindex];
1129	conn->writesockfd = writesockindex == -1 ?
1130	CURL_SOCKET_BAD:conn->sock[writesockindex];
1131	}
1132	k->getheader = getheader;
1133
1134	k->size = size;
1135
1136	/* The code sequence below is placed in this function just because all
1137	necessary input is not always known in do_complete() as this function may
1138	be called after that */
1139
1140	if(!k->getheader) {
1141	k->header = FALSE;
1142	if(size > 0)
1143	Curl_pgrsSetDownloadSize(data, size);
1144	}
1145	/* we want header and/or body, if neither then don't do this! */
1146	if(k->getheader \|\| !data->req.no_body) {
1147
1148	if(sockindex != -1)
1149	k->keepon \|= KEEP_RECV;
1150
1151	if(writesockindex != -1)
1152	k->keepon \|= KEEP_SEND;
1153	} /* if(k->getheader \|\| !data->req.no_body) */
1154
1155	}
1156
1157	CURLcode Curl_xfer_write_resp(struct Curl_easy *data,
1158	char *buf, size_t blen,
1159	bool is_eos)
1160	{
1161	CURLcode result = CURLE_OK;
1162
1163	if(data->conn->handler->write_resp) {
1164	/* protocol handlers offering this function take full responsibility
1165	* for writing all received download data to the client. */
1166	result = data->conn->handler->write_resp(data, buf, blen, is_eos);
1167	}
1168	else {
1169	/* No special handling by protocol handler, write all received data
1170	* as BODY to the client. */
1171	if(blen \|\| is_eos) {
1172	int cwtype = CLIENTWRITE_BODY;
1173	if(is_eos)
1174	cwtype \|= CLIENTWRITE_EOS;
1175
1176	#ifndef CURL_DISABLE_POP3
1177	if(blen && data->conn->handler->protocol & PROTO_FAMILY_POP3) {
1178	result = data->req.ignorebody? CURLE_OK :
1179	Curl_pop3_write(data, buf, blen);
1180	}
1181	else
1182	#endif /* CURL_DISABLE_POP3 */
1183	result = Curl_client_write(data, cwtype, buf, blen);
1184	}
1185	}
1186
1187	if(!result && is_eos) {
1188	/* If we wrote the EOS, we are definitely done */
1189	data->req.eos_written = TRUE;
1190	data->req.download_done = TRUE;
1191	}
1192	return result;
1193	}
1194
1195	CURLcode Curl_xfer_write_done(struct Curl_easy *data, bool premature)
1196	{
1197	(void)premature;
1198	return Curl_cw_out_done(data);
1199	}
1200
1201	CURLcode Curl_xfer_send(struct Curl_easy *data,
1202	const void *buf, size_t blen,
1203	size_t *pnwritten)
1204	{
1205	CURLcode result;
1206	int sockindex;
1207
1208	if(!data \|\| !data->conn)
1209	return CURLE_FAILED_INIT;
1210	/* FIXME: would like to enable this, but some protocols (MQTT) do not
1211	* setup the transfer correctly, it seems
1212	if(data->conn->writesockfd == CURL_SOCKET_BAD) {
1213	failf(data, "transfer not setup for sending");
1214	DEBUGASSERT(0);
1215	return CURLE_SEND_ERROR;
1216	} */
1217	sockindex = ((data->conn->writesockfd != CURL_SOCKET_BAD) &&
1218	(data->conn->writesockfd == data->conn->sock[SECONDARYSOCKET]));
1219	result = Curl_conn_send(data, sockindex, buf, blen, pnwritten);
1220	if(result == CURLE_AGAIN) {
1221	result = CURLE_OK;
1222	*pnwritten = 0;
1223	}
1224	return result;
1225	}
1226
1227	CURLcode Curl_xfer_recv(struct Curl_easy *data,
1228	char *buf, size_t blen,
1229	ssize_t *pnrcvd)
1230	{
1231	int sockindex;
1232
1233	if(!data \|\| !data->conn)
1234	return CURLE_FAILED_INIT;
1235	/* FIXME: would like to enable this, but some protocols (MQTT) do not
1236	* setup the transfer correctly, it seems
1237	if(data->conn->sockfd == CURL_SOCKET_BAD) {
1238	failf(data, "transfer not setup for receiving");
1239	DEBUGASSERT(0);
1240	return CURLE_RECV_ERROR;
1241	} */
1242	sockindex = ((data->conn->sockfd != CURL_SOCKET_BAD) &&
1243	(data->conn->sockfd == data->conn->sock[SECONDARYSOCKET]));
1244	if(data->set.buffer_size > 0 && (size_t)data->set.buffer_size < blen)
1245	blen = (size_t)data->set.buffer_size;
1246	return Curl_conn_recv(data, sockindex, buf, blen, pnrcvd);
1247	}
1248
1249	CURLcode Curl_xfer_send_close(struct Curl_easy *data)
1250	{
1251	Curl_conn_ev_data_done_send(data);
1252	return CURLE_OK;
1253	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/libs/curl-8.7.1/lib/transfer.c@ 104083

Download in other formats: