ssl-openssl.cpp@ 95381

Last change on this file since 95381 was 95381, checked in by vboxsync, 2 years ago
Runtime/ssl-openssl: Free the session in the error path, bugref:3409
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 15.8 KB

Line
1	/** @file
2	* IPRT - Crypto - Secure Socket Layer (SSL) / Transport Security Layer (TLS).
3	*/
4
5	/*
6	* Copyright (C) 2018-2022 Oracle Corporation
7	*
8	* This file is part of VirtualBox Open Source Edition (OSE), as
9	* available from http://www.virtualbox.org. This file is free software;
10	* you can redistribute it and/or modify it under the terms of the GNU
11	* General Public License (GPL) as published by the Free Software
12	* Foundation, in version 2 as it comes in the "COPYING" file of the
13	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15	*
16	* The contents of this file may alternatively be used under the terms
17	* of the Common Development and Distribution License Version 1.0
18	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19	* VirtualBox OSE distribution, in which case the provisions of the
20	* CDDL are applicable instead of those of the GPL.
21	*
22	* You may elect to license modified versions of this file under the
23	* terms and conditions of either the GPL or the CDDL or both.
24	*/
25	#ifdef IPRT_WITH_OPENSSL /* whole file */
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	# include "internal/iprt.h"
32	# include <iprt/crypto/ssl.h>
33
34	# include <iprt/asm.h>
35	# include <iprt/assert.h>
36	# include <iprt/err.h>
37	# include <iprt/file.h>
38	# include <iprt/mem.h>
39	# include <iprt/string.h>
40
41	# include "internal/magics.h"
42
43	# include "internal/iprt-openssl.h"
44	# include "internal/openssl-pre.h"
45	# include <openssl/ssl.h>
46	# include <openssl/tls1.h>
47	# include "internal/openssl-post.h"
48
49
50	/*********************************************************************************************************************************
51	* Header Files *
52	*********************************************************************************************************************************/
53	/**
54	* SSL instance data for OpenSSL.
55	*/
56	typedef struct RTCRSSLINT
57	{
58	/** Magic value (RTCRSSLINT_MAGIC). */
59	uint32_t u32Magic;
60	/** Reference count. */
61	uint32_t volatile cRefs;
62	/** The SSL context. */
63	SSL_CTX *pCtx;
64	} RTCRSSLINT;
65
66	/**
67	* SSL session instance data for OpenSSL.
68	*/
69	typedef struct RTCRSSLSESSIONINT
70	{
71	/** Magic value (RTCRSSLSESSIONINT_MAGIC). */
72	uint32_t u32Magic;
73	/** Reference count. */
74	uint32_t volatile cRefs;
75	/** RTCRSSLSESSION_F_XXX. */
76	uint32_t fFlags;
77
78	/** The SSL instance. */
79	SSL *pSsl;
80	/** The socket BIO instance. */
81	BIO *pBio;
82	} RTCRSSLSESSIONINT;
83
84
85
86	RTDECL(int) RTCrSslCreate(PRTCRSSL phSsl, uint32_t fFlags)
87	{
88	AssertPtr(phSsl);
89	*phSsl = NIL_RTCRSSL;
90	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
91
92	SSL_library_init();
93
94	/*
95	* We aim at TLSv1 or higher here by default.
96	*/
97	# if OPENSSL_VERSION_NUMBER >= 0x10100000
98	const SSL_METHOD *pSslMethod = TLS_method();
99	# elif OPENSSL_VERSION_NUMBER >= 0x10002000
100	const SSL_METHOD *pSslMethod = SSLv23_method();
101	# elif OPENSSL_VERSION_NUMBER >= 0x10000000
102	const SSL_METHOD *pSslMethod = TLSv1_method();
103	# else
104	SSL_METHOD *pSslMethod = TLSv1_method();
105	# endif
106	if (pSslMethod)
107	{
108	RTCRSSLINT pThis = (RTCRSSLINT )RTMemAllocZ(sizeof(*pThis));
109	if (pThis)
110	{
111	pThis->pCtx = SSL_CTX_new(pSslMethod);
112	if (pThis->pCtx)
113	{
114	/* Help with above aim. */
115	# if OPENSSL_VERSION_NUMBER >= 0x10100000
116	# ifndef SSL_CTX_get_min_proto_version
117	/* Some older OpenSSL 1.1.0 releases lack the getters, officially they were
118	* added with 1.1.1 but someone cherry picked them, just maybe too late. */
119	# define SSL_CTX_get_min_proto_version(ctx) (0)
120	# endif
121	if (SSL_CTX_get_min_proto_version(pThis->pCtx) < TLS1_VERSION)
122	SSL_CTX_set_min_proto_version(pThis->pCtx, TLS1_VERSION);
123	# elif OPENSSL_VERSION_NUMBER >= 0x10002000
124	SSL_CTX_set_options(pThis->pCtx, SSL_OP_NO_SSLv2);
125	SSL_CTX_set_options(pThis->pCtx, SSL_OP_NO_SSLv3);
126	# endif
127
128	/*
129	* Complete the instance and return it.
130	*/
131	pThis->u32Magic = RTCRSSLINT_MAGIC;
132	pThis->cRefs = 1;
133
134	*phSsl = pThis;
135	return VINF_SUCCESS;
136	}
137
138	RTMemFree(pThis);
139	}
140	return VERR_NO_MEMORY;
141	}
142	return VERR_NOT_SUPPORTED;
143	}
144
145
146	RTDECL(uint32_t) RTCrSslRetain(RTCRSSL hSsl)
147	{
148	RTCRSSLINT *pThis = hSsl;
149	AssertPtrReturn(pThis, UINT32_MAX);
150	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, UINT32_MAX);
151
152	uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
153	Assert(cRefs > 1);
154	Assert(cRefs < 1024);
155	return cRefs;
156	}
157
158
159	/**
160	* Worker for RTCrSslRelease.
161	*/
162	static int rtCrSslDestroy(RTCRSSLINT *pThis)
163	{
164	ASMAtomicWriteU32(&pThis->u32Magic, ~RTCRSSLINT_MAGIC);
165	SSL_CTX_free(pThis->pCtx);
166	pThis->pCtx = NULL;
167	RTMemFree(pThis);
168	return 0;
169	}
170
171
172	RTDECL(uint32_t) RTCrSslRelease(RTCRSSL hSsl)
173	{
174	RTCRSSLINT *pThis = hSsl;
175	if (pThis == NIL_RTCRSSL)
176	return 0;
177	AssertPtrReturn(pThis, UINT32_MAX);
178	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, UINT32_MAX);
179
180	uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
181	Assert(cRefs < 1024);
182	if (cRefs == 0)
183	return rtCrSslDestroy(pThis);
184	return cRefs;
185	}
186
187
188	RTDECL(int) RTCrSslSetCertificateFile(RTCRSSL hSsl, const char *pszFile, uint32_t fFlags)
189	{
190	RTCRSSLINT *pThis = hSsl;
191	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
192	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
193	AssertReturn(!(fFlags & ~RTCRSSL_FILE_F_ASN1), VERR_INVALID_FLAGS);
194
195	int rcOssl = SSL_CTX_use_certificate_file(pThis->pCtx, pszFile,
196	RTCRSSL_FILE_F_ASN1 & fFlags ? SSL_FILETYPE_ASN1 : SSL_FILETYPE_PEM);
197	if (rcOssl != 0)
198	return VINF_SUCCESS;
199	return !pszFile \|\| !pszFile \|\| !RTFileExists(pszFile) ? VERR_FILE_NOT_FOUND : VERR_OPEN_FAILED; /* @todo Better status codes */
200	}
201
202
203	RTDECL(int) RTCrSslSetPrivateKeyFile(RTCRSSL hSsl, const char *pszFile, uint32_t fFlags)
204	{
205	RTCRSSLINT *pThis = hSsl;
206	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
207	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
208	AssertReturn(!(fFlags & ~RTCRSSL_FILE_F_ASN1), VERR_INVALID_FLAGS);
209
210	int rcOssl = SSL_CTX_use_PrivateKey_file(pThis->pCtx, pszFile,
211	RTCRSSL_FILE_F_ASN1 & fFlags ? SSL_FILETYPE_ASN1 : SSL_FILETYPE_PEM);
212	if (rcOssl != 0)
213	return VINF_SUCCESS;
214	return !pszFile \|\| !pszFile \|\| !RTFileExists(pszFile) ? VERR_FILE_NOT_FOUND : VERR_OPEN_FAILED; /* @todo Better status codes */
215	}
216
217
218	RTDECL(int) RTCrSslLoadTrustedRootCerts(RTCRSSL hSsl, const char pszFile, const char pszDir)
219	{
220	RTCRSSLINT *pThis = hSsl;
221	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
222	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
223
224	int rcOssl = SSL_CTX_load_verify_locations(pThis->pCtx, pszFile, pszDir);
225	if (rcOssl != 0)
226	return VINF_SUCCESS;
227
228	if (!pszFile \|\| !*pszFile \|\| !RTFileExists(pszFile))
229	return VERR_FILE_NOT_FOUND;
230	return VERR_OPEN_FAILED; /** @todo Better status codes */
231	}
232
233
234	RTDECL(int) RTCrSslSetNoPeerVerify(RTCRSSL hSsl)
235	{
236	RTCRSSLINT *pThis = hSsl;
237	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
238	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
239
240	SSL_CTX_set_verify(pThis->pCtx, SSL_VERIFY_NONE, NULL);
241	return VINF_SUCCESS;
242	}
243
244
245
246	//RTDECL(int) RTCrSslCreateSession(RTCRSSL hSsl, RTSOCKET hSocket, uint32_t fFlags, PRTCRSSLSESSION phSslConn);
247
248	RTDECL(int) RTCrSslCreateSessionForNativeSocket(RTCRSSL hSsl, RTHCINTPTR hNativeSocket, uint32_t fFlags,
249	PRTCRSSLSESSION phSslSession)
250	{
251	/*
252	* Validate input.
253	*/
254	*phSslSession = NIL_RTCRSSLSESSION;
255
256	RTCRSSLINT *pThis = hSsl;
257	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
258	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
259	AssertReturn(!(fFlags & ~RTCRSSLSESSION_F_NON_BLOCKING), VERR_INVALID_FLAGS);
260
261	/*
262	* Create a new session.
263	*/
264	int rc = VERR_NO_MEMORY;
265	RTCRSSLSESSIONINT pSession = (RTCRSSLSESSIONINT )RTMemAllocZ(sizeof(*pSession));
266	if (pSession)
267	{
268	pSession->pSsl = SSL_new(pThis->pCtx);
269	if (pSession->pSsl)
270	{
271	/* Disable read-ahead if non-blocking socket relying on select/poll. */
272	if (fFlags & RTCRSSLSESSION_F_NON_BLOCKING)
273	SSL_set_read_ahead(pSession->pSsl, 0);
274
275	/* Create a wrapper for the socket handle. */
276	pSession->pBio = BIO_new_socket(hNativeSocket, BIO_NOCLOSE);
277	if (pSession->pBio)
278	{
279	# if (OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER)) \|\| LIBRESSL_VERSION_NUMBER >= 0x2070000f
280	BIO_up_ref(pSession->pBio); /* our reference. */
281	# endif
282	SSL_set_bio(pSession->pSsl, pSession->pBio, pSession->pBio);
283
284	/*
285	* Done.
286	*/
287	pSession->cRefs = 1;
288	pSession->u32Magic = RTCRSSLSESSIONINT_MAGIC;
289	*phSslSession = pSession;
290	return VINF_SUCCESS;
291	}
292
293	SSL_free(pSession->pSsl);
294	pSession->pSsl = NULL;
295	}
296	RTMemFree(pSession);
297	}
298	return rc;
299	}
300
301
302	/*********************************************************************************************************************************
303	* Session implementation. *
304	*********************************************************************************************************************************/
305
306	RTDECL(uint32_t) RTCrSslSessionRetain(RTCRSSLSESSION hSslSession)
307	{
308	RTCRSSLSESSIONINT *pThis = hSslSession;
309	AssertPtrReturn(pThis, UINT32_MAX);
310	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, UINT32_MAX);
311
312	uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
313	Assert(cRefs > 1);
314	Assert(cRefs < 1024);
315	return cRefs;
316	}
317
318
319	/**
320	* Worker for RTCrSslRelease.
321	*/
322	static int rtCrSslSessionDestroy(RTCRSSLSESSIONINT *pThis)
323	{
324	ASMAtomicWriteU32(&pThis->u32Magic, ~RTCRSSLSESSIONINT_MAGIC);
325	SSL_free(pThis->pSsl);
326	pThis->pSsl = NULL;
327	# if (OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER)) \|\| LIBRESSL_VERSION_NUMBER >= 0x2070000f
328	BIO_free(pThis->pBio);
329	# endif
330	pThis->pBio = NULL;
331	RTMemFree(pThis);
332	return 0;
333	}
334
335
336	RTDECL(uint32_t) RTCrSslSessionRelease(RTCRSSLSESSION hSslSession)
337	{
338	RTCRSSLSESSIONINT *pThis = hSslSession;
339	if (pThis == NIL_RTCRSSLSESSION)
340	return 0;
341	AssertPtrReturn(pThis, UINT32_MAX);
342	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, UINT32_MAX);
343
344	uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
345	Assert(cRefs < 1024);
346	if (cRefs == 0)
347	return rtCrSslSessionDestroy(pThis);
348	return cRefs;
349	}
350
351
352	RTDECL(int) RTCrSslSessionAccept(RTCRSSLSESSION hSslSession, uint32_t fFlags)
353	{
354	RTCRSSLSESSIONINT *pThis = hSslSession;
355	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
356	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
357	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
358
359	int rcOssl = SSL_accept(pThis->pSsl);
360	if (rcOssl > 0)
361	return VINF_SUCCESS;
362
363	/** @todo better status codes. */
364	if (BIO_should_retry(pThis->pBio))
365	return VERR_TRY_AGAIN;
366	return VERR_NOT_SUPPORTED;
367	}
368
369
370	RTDECL(int) RTCrSslSessionConnect(RTCRSSLSESSION hSslSession, uint32_t fFlags)
371	{
372	RTCRSSLSESSIONINT *pThis = hSslSession;
373	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
374	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
375	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
376
377	int rcOssl = SSL_connect(pThis->pSsl);
378	if (rcOssl > 0)
379	return VINF_SUCCESS;
380
381	/** @todo better status codes. */
382	if (BIO_should_retry(pThis->pBio))
383	return VERR_TRY_AGAIN;
384	return VERR_NOT_SUPPORTED;
385	}
386
387
388	RTDECL(const char *) RTCrSslSessionGetVersion(RTCRSSLSESSION hSslSession)
389	{
390	RTCRSSLSESSIONINT *pThis = hSslSession;
391	AssertPtrReturn(pThis, NULL);
392	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, NULL);
393
394	return SSL_get_version(pThis->pSsl);
395	}
396
397
398	RTDECL(int) RTCrSslSessionGetCertIssuerNameAsString(RTCRSSLSESSION hSslSession, char pszBuf, size_t cbBuf, size_t pcbActual)
399	{
400	RTCRSSLSESSIONINT *pThis = hSslSession;
401	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
402	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
403	AssertPtrNull(pszBuf);
404	AssertPtrNull(pcbActual);
405	if (pcbActual)
406	*pcbActual = 0;
407
408	/*
409	* Get and format the certificate issuer name.
410	*/
411	int rc = VERR_NOT_AVAILABLE;
412	X509 *pCert = SSL_get_certificate(pThis->pSsl);
413	if (pCert)
414	{
415	X509_NAME *pIssuer = X509_get_issuer_name(pCert);
416	if (pIssuer)
417	{
418	char *pszSrc = X509_NAME_oneline(pIssuer, NULL, 0);
419	if (pszSrc)
420	{
421	/*
422	* Copy out the result and free it.
423	*/
424	size_t cbNeeded = strlen(pszSrc) + 1;
425	if (pcbActual)
426	*pcbActual = cbNeeded;
427
428	if (pszBuf != NULL && cbBuf > 0)
429	{
430	if (cbBuf >= cbNeeded)
431	{
432	memcpy(pszBuf, pszSrc, cbNeeded);
433	rc = VINF_SUCCESS;
434	}
435	else
436	{
437	memcpy(pszBuf, pszSrc, cbBuf - 1);
438	pszBuf[cbBuf - 1] = '\0';
439	rc = VERR_BUFFER_OVERFLOW;
440	}
441	}
442	else
443	rc = VERR_BUFFER_OVERFLOW;
444	OPENSSL_free(pszSrc);
445	}
446	}
447	}
448	return rc;
449	}
450
451
452	RTDECL(bool) RTCrSslSessionPending(RTCRSSLSESSION hSslSession)
453	{
454	RTCRSSLSESSIONINT *pThis = hSslSession;
455	AssertPtrReturn(pThis, true);
456	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, true);
457
458	return SSL_pending(pThis->pSsl) != 0;
459	}
460
461
462	RTDECL(ssize_t) RTCrSslSessionRead(RTCRSSLSESSION hSslSession, void *pvBuf, size_t cbToRead)
463	{
464	RTCRSSLSESSIONINT *pThis = hSslSession;
465	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
466	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
467
468	Assert((size_t)(int)cbToRead == cbToRead);
469
470	int cbActual = SSL_read(pThis->pSsl, pvBuf, (int)cbToRead);
471	if (cbActual > 0)
472	return cbActual;
473	if (BIO_should_retry(pThis->pBio))
474	return VERR_TRY_AGAIN;
475	return VERR_READ_ERROR; /** @todo better status codes. */
476	}
477
478
479	RTDECL(ssize_t) RTCrSslSessionWrite(RTCRSSLSESSION hSslSession, void const *pvBuf, size_t cbToWrite)
480	{
481	RTCRSSLSESSIONINT *pThis = hSslSession;
482	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
483	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
484
485	Assert((size_t)(int)cbToWrite == cbToWrite);
486	Assert(cbToWrite != 0 /* undefined behavior if zero */);
487
488	int cbActual = SSL_write(pThis->pSsl, pvBuf, (int)cbToWrite);
489	if (cbActual > 0)
490	return cbActual;
491	if (BIO_should_retry(pThis->pBio))
492	return VERR_TRY_AGAIN;
493	return VERR_WRITE_ERROR; /** @todo better status codes. */
494	}
495
496	#endif /* IPRT_WITH_OPENSSL */
497

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/crypto/ssl-openssl.cpp@ 95381

Download in other formats: