From 74d36208392415870b9e071155562f1f3d87af4b Mon Sep 17 00:00:00 2001 From: vylsaz Date: Sat, 3 Feb 2024 01:45:56 +0000 Subject: [PATCH 1/3] =?UTF-8?q?Implement=20=E2=80=A2SH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/sysfn.c | 94 +++++++++++++++++++++ src/windows/sh.c | 195 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+) create mode 100644 src/windows/sh.c diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index e7f0b792..be3677c6 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -1169,6 +1169,100 @@ B sh_c2(B t, B w, B x) { : -1; return m_hvec3(m_i32(code), s_outObj, s_errObj); } +#elif defined(_WIN32) || defined(_WIN64) +#define HAS_SH 1 +#include "../windows/winError.c" +#include "../windows/sh.c" + +B sh_c2(B t, B w, B x) { + + // parse options + B inObj = bi_N; + bool raw = false; + if (!q_N(w)) { + if (!isNsp(w)) thrM("•SH: 𝕨 must be a namespace"); + inObj = ns_getC(w, "stdin"); + if (!q_N(inObj) && !isArr(inObj)) thrM("•SH: Invalid stdin value"); + B rawObj = ns_getC(w, "raw"); + if (!q_N(rawObj)) raw = o2b(rawObj); + } + u64 iLen = q_N(inObj)? 0 : (raw? IA(inObj) : utf8lenB(inObj)); + + // allocate args + if (isAtm(x) || RNK(x)>1) thrM("•SH: 𝕩 must be a vector of strings"); + usz xia = IA(x); + if (xia==0) thrM("•SH: 𝕩 must have at least one item"); + u64 arglen = 0; + SGetU(x) + for (u64 i = 0; i < xia; i++) { + B c = GetU(x, i); + if (isAtm(c) || RNK(c)!=1) thrM("•SH: 𝕩 must be a vector of strings"); + u64 len = utf8lenB(c); + arglen += 1+2+2*len; + // space or 0, quotes, worst-case scenario (every character needs escaping) + } + TALLOC(char, arg, arglen); + char* pos = arg; + for (u64 i = 0; i < xia; i++) { + B c = GetU(x, i); + u64 len = utf8lenB(c); + TALLOC(char, cstr, len+1); + toUTF8(c, cstr); + cstr[len] = 0; + pos = winQuoteCmdArg(len, cstr, pos); + *(pos++) = (xia==i+1)? '\0' : ' '; + TFREE(cstr) + assert(pos <= arg+arglen); + } + + // allocate stdin + char* iBuf; + CharBuf iBufRaw; + if (iLen>0) { + if (raw) { + iBufRaw = get_chars(inObj); + iBuf = iBufRaw.data; + } else { + iBuf = TALLOCP(char, iLen); + toUTF8(inObj, iBuf); + } + } else iBuf = ""; + + DWORD code = -1; + u64 oLen = 0; char* oBuf; + u64 eLen = 0; char* eBuf; + DWORD dwResult = winCmd(arg, iLen, iBuf, &code, &oLen, &oBuf, &eLen, &eBuf); + if (iLen>0) { if (raw) free_chars(iBufRaw); else TFREE(iBuf); } // FREE_INPUT + TFREE(arg) + if (dwResult != ERROR_SUCCESS) { + thrF("•SH: Failed to run command: %S", winErrorEx(dwResult)); + } + + u8* op; + B s_out = m_c8arrv(&op, oLen); + if (oLen > 0 && oBuf != NULL) { + memcpy(op, oBuf, oLen*sizeof(char)); free(oBuf); + } + u8* ep; + B s_err = m_c8arrv(&ep, eLen); + if (eLen > 0 && eBuf != NULL) { + memcpy(ep, eBuf, eLen*sizeof(char)); free(eBuf); + } + + dec(w); dec(x); + B s_outRaw = toC8Any(s_out); + B s_errRaw = toC8Any(s_err); + B s_outObj; + B s_errObj; + if (raw) { + s_outObj = s_outRaw; + s_errObj = s_errRaw; + } else { + s_outObj = utf8Decode((char*)c8any_ptr(s_outRaw), IA(s_outRaw)); dec(s_outRaw); + s_errObj = utf8Decode((char*)c8any_ptr(s_errRaw), IA(s_errRaw)); dec(s_errRaw); + } + return m_hvec3(m_i32((i32)code), s_outObj, s_errObj); +} #else #if FOR_BUILD #if __ANDROID__ diff --git a/src/windows/sh.c b/src/windows/sh.c new file mode 100644 index 00000000..763f4ffe --- /dev/null +++ b/src/windows/sh.c @@ -0,0 +1,195 @@ +#include + +// https://github.com/libuv/libuv/blob/v1.23.0/src/win/process.c#L454-L524 + +static char* winQuoteCmdArg(u64 len, char* source, char* target) { + if (len == 0) { + // Need double quotation for empty argument + *(target++) = '"'; + *(target++) = '"'; + return target; + } + if (NULL == strpbrk(source, " \t\"")) { + // No quotation needed + memcpy(target, source, len * sizeof(char)); target += len; + return target; + } + if (NULL == strpbrk(source, "\"\\")) { + // No embedded double quotes or backlashes, so I can just wrap + // quote marks around the whole thing. + *(target++) = '"'; + memcpy(target, source, len * sizeof(char)); target += len; + *(target++) = '"'; + return target; + } + + *(target++) = '"'; + char *start = target; + int quote_hit = 1; + + for (u64 i = 0; i < len; ++i) { + *(target++) = source[len - 1 - i]; + + if (quote_hit && source[len - 1 - i] == '\\') { + *(target++) = '\\'; + } else if (source[len - 1 - i] == '"') { + quote_hit = 1; + *(target++) = '\\'; + } else { + quote_hit = 0; + } + } + target[0] = '\0'; _strrev(start); + *(target++) = '"'; + return target; +} + +char *dataInp, *dataOut, *dataErr; +u64 sizeInp, sizeOut, sizeErr; + +static DWORD WINAPI winThreadWriteIn(LPVOID arg) { + DWORD dwResult = ERROR_SUCCESS; + HANDLE hndl = (HANDLE)arg; + char *wBuf = dataInp; + DWORD dwToWrite = sizeInp, dwWritten = 0, dwOff = 0; + + for (;;) { + BOOL bOk = WriteFile(hndl, &wBuf[dwOff], dwToWrite-dwOff, &dwWritten, NULL); + if (!bOk) { dwResult = GetLastError(); break; } + dwOff += dwWritten; + if (dwOff >= dwToWrite) { break; } + } + + CloseHandle(hndl); + return dwResult; +} + +#define winThreadRead(D) \ +static DWORD WINAPI winThreadRead##D(LPVOID arg) { \ + DWORD dwResult = ERROR_SUCCESS; \ + HANDLE hndl = (HANDLE)arg; \ + u8 buf[1024] = {0}; \ + const usz bufSize = sizeof(buf)/sizeof(u8); \ + DWORD dwRead = 0, dwHasRead = 0; \ + char *rBuf = NULL; \ + for (;;) { \ + ZeroMemory(buf, bufSize); \ + BOOL bOk = ReadFile(hndl, buf, bufSize, &dwRead, NULL); \ + if (!bOk) { \ + DWORD dwErr = GetLastError(); \ + if (dwErr == ERROR_BROKEN_PIPE) { break; } \ + else { dwResult = dwErr; break; } \ + } \ + char *newBuf = (rBuf==NULL)? \ + calloc(dwHasRead+dwRead, sizeof(char)) : \ + realloc(rBuf, (dwHasRead+dwRead)*sizeof(char)); \ + if (newBuf == NULL) { dwResult = GetLastError(); break; } \ + rBuf = newBuf; \ + memcpy(&rBuf[dwHasRead], buf, dwRead); \ + dwHasRead += dwRead; \ + } \ + data##D = rBuf; \ + size##D = dwHasRead; \ + CloseHandle(hndl); \ + return dwResult; \ +} + +winThreadRead(Out) +winThreadRead(Err) + +#undef winThreadRead + +static DWORD winCmd(char* arg, + u64 iLen, char* iBuf, + DWORD* code, + u64* oLen, char** oBuf, + u64* eLen, char** eBuf) { + + DWORD dwResult = ERROR_SUCCESS; + HANDLE hInpR, hInpW; + HANDLE hOutR, hOutW; + HANDLE hErrR, hErrW; + + // Create pipes + SECURITY_ATTRIBUTES sa; + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + sa.lpSecurityDescriptor = NULL; + sa.bInheritHandle = TRUE; + + CreatePipe(&hInpR, &hInpW, &sa, 0); + CreatePipe(&hOutR, &hOutW, &sa, 0); + CreatePipe(&hErrR, &hErrW, &sa, 0); + + SetHandleInformation(hInpW, HANDLE_FLAG_INHERIT, 0); + SetHandleInformation(hOutR, HANDLE_FLAG_INHERIT, 0); + SetHandleInformation(hErrR, HANDLE_FLAG_INHERIT, 0); + + // Set up + STARTUPINFO si; + ZeroMemory(&si, sizeof(STARTUPINFO)); + si.cb = sizeof(STARTUPINFO); + si.hStdInput = hInpR; + si.hStdOutput = hOutW; + si.hStdError = hErrW; + si.dwFlags |= STARTF_USESTDHANDLES | STARTF_UNTRUSTEDSOURCE; + + PROCESS_INFORMATION pi; + ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); + + // Create the child process + BOOL bSuccess = CreateProcessA(NULL, arg, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi); + if (!bSuccess) { return GetLastError(); } + + // Close the unneeded handles + CloseHandle(hInpR); + CloseHandle(hOutW); + CloseHandle(hErrW); + + // Now we are dealing with the globals and threads + + + + dataInp = iBuf; + sizeInp = iLen; + + DWORD exitCode = -1; + HANDLE lpThreads[3]; + lpThreads[0] = CreateThread(NULL, 0, winThreadWriteIn, (LPVOID)hInpW, 0, NULL); + lpThreads[1] = CreateThread(NULL, 0, winThreadReadOut, (LPVOID)hOutR, 0, NULL); + lpThreads[2] = CreateThread(NULL, 0, winThreadReadErr, (LPVOID)hErrR, 0, NULL); + + for (int i = 0; i < 3; ++i) { + if (lpThreads[i] == NULL) { dwResult = GetLastError(); goto error; } + } + + // Wait for the threads + if (WaitForMultipleObjects(3, lpThreads, TRUE, INFINITE) == WAIT_FAILED) { + dwResult = GetLastError(); goto error; + } + for (int i = 0; i < 3; ++i) { + GetExitCodeThread(lpThreads[i], &exitCode); + if (exitCode != ERROR_SUCCESS) { dwResult = exitCode; goto error; } + } + + // Get the exit code + if (WaitForSingleObject(pi.hProcess, INFINITE) == WAIT_FAILED) { + dwResult = GetLastError(); goto error; + } + exitCode = -1; + GetExitCodeProcess(pi.hProcess, &exitCode); + + // Give outputs + *code = exitCode; + *oLen = sizeOut; *oBuf = dataOut; + *eLen = sizeErr; *eBuf = dataErr; + +error: + // Close handles + for (int i = 0; i < 3; ++i) { CloseHandle(lpThreads[i]); } + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + + + + return dwResult; +} From 42e4956fb97b3878b725296d4ca1fc4e60c67d39 Mon Sep 17 00:00:00 2001 From: vylsaz Date: Sun, 4 Feb 2024 02:35:47 +0000 Subject: [PATCH 2/3] Replaced globals with argument to thread procedure --- src/builtins/sysfn.c | 6 ++- src/windows/sh.c | 107 ++++++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index be3677c6..e558a61f 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -1120,7 +1120,7 @@ B sh_c2(B t, B w, B x) { bool any = false; if (ps[out_i].revents & POLLIN) while(true) { i64 len = read(p_out[0], &oBuf[0], bufsz); shDbg("read stdout "N64d"\n",len); if(len<=0) break; else any=true; *oBufIA = len; s_out = vec_join(s_out, incG(oBufObj)); } if (ps[err_i].revents & POLLIN) while(true) { i64 len = read(p_err[0], &oBuf[0], bufsz); shDbg("read stderr "N64d"\n",len); if(len<=0) break; else any=true; *oBufIA = len; s_err = vec_join(s_err, incG(oBufObj)); } - if (!iDone && ps[in_i].revents & POLLOUT) { + if (!iDone && ps[in_i].revents & POLLOUT) { shDbg("writing "N64u"\n", iLen-iOff); ssize_t ww = write(p_in[1], iBuf+iOff, iLen-iOff); shDbg("written %zd/"N64u"\n", ww, iLen-iOff); @@ -1228,16 +1228,18 @@ B sh_c2(B t, B w, B x) { } } else iBuf = ""; + // run command DWORD code = -1; u64 oLen = 0; char* oBuf; u64 eLen = 0; char* eBuf; DWORD dwResult = winCmd(arg, iLen, iBuf, &code, &oLen, &oBuf, &eLen, &eBuf); if (iLen>0) { if (raw) free_chars(iBufRaw); else TFREE(iBuf); } // FREE_INPUT TFREE(arg) - if (dwResult != ERROR_SUCCESS) { + if (dwResult != ERROR_SUCCESS) { thrF("•SH: Failed to run command: %S", winErrorEx(dwResult)); } + // prepare output u8* op; B s_out = m_c8arrv(&op, oLen); if (oLen > 0 && oBuf != NULL) { diff --git a/src/windows/sh.c b/src/windows/sh.c index 763f4ffe..56654a53 100644 --- a/src/windows/sh.c +++ b/src/windows/sh.c @@ -44,14 +44,18 @@ static char* winQuoteCmdArg(u64 len, char* source, char* target) { return target; } -char *dataInp, *dataOut, *dataErr; -u64 sizeInp, sizeOut, sizeErr; +typedef struct { + HANDLE hndl; + char* buf; + u64 len; +} ThreadIO; -static DWORD WINAPI winThreadWriteIn(LPVOID arg) { +static DWORD WINAPI winThreadWrite(LPVOID arg0) { DWORD dwResult = ERROR_SUCCESS; - HANDLE hndl = (HANDLE)arg; - char *wBuf = dataInp; - DWORD dwToWrite = sizeInp, dwWritten = 0, dwOff = 0; + ThreadIO* arg = arg0; + HANDLE hndl = arg->hndl; + char* wBuf = arg->buf; + DWORD dwToWrite = arg->len, dwWritten = 0, dwOff = 0; for (;;) { BOOL bOk = WriteFile(hndl, &wBuf[dwOff], dwToWrite-dwOff, &dwWritten, NULL); @@ -64,41 +68,42 @@ static DWORD WINAPI winThreadWriteIn(LPVOID arg) { return dwResult; } -#define winThreadRead(D) \ -static DWORD WINAPI winThreadRead##D(LPVOID arg) { \ - DWORD dwResult = ERROR_SUCCESS; \ - HANDLE hndl = (HANDLE)arg; \ - u8 buf[1024] = {0}; \ - const usz bufSize = sizeof(buf)/sizeof(u8); \ - DWORD dwRead = 0, dwHasRead = 0; \ - char *rBuf = NULL; \ - for (;;) { \ - ZeroMemory(buf, bufSize); \ - BOOL bOk = ReadFile(hndl, buf, bufSize, &dwRead, NULL); \ - if (!bOk) { \ - DWORD dwErr = GetLastError(); \ - if (dwErr == ERROR_BROKEN_PIPE) { break; } \ - else { dwResult = dwErr; break; } \ - } \ - char *newBuf = (rBuf==NULL)? \ - calloc(dwHasRead+dwRead, sizeof(char)) : \ - realloc(rBuf, (dwHasRead+dwRead)*sizeof(char)); \ - if (newBuf == NULL) { dwResult = GetLastError(); break; } \ - rBuf = newBuf; \ - memcpy(&rBuf[dwHasRead], buf, dwRead); \ - dwHasRead += dwRead; \ - } \ - data##D = rBuf; \ - size##D = dwHasRead; \ - CloseHandle(hndl); \ - return dwResult; \ +static DWORD WINAPI winThreadRead(LPVOID arg0) { + DWORD dwResult = ERROR_SUCCESS; + ThreadIO* arg = arg0; + HANDLE hndl = arg->hndl; + u8 buf[1024] = {0}; + const usz bufSize = sizeof(buf)/sizeof(u8); + DWORD dwRead = 0, dwHasRead = 0; + char* rBuf = NULL; + + for (;;) { + ZeroMemory(buf, bufSize); + BOOL bOk = ReadFile(hndl, buf, bufSize, &dwRead, NULL); + if (!bOk) { + DWORD dwErr = GetLastError(); + if (dwErr == ERROR_BROKEN_PIPE) { break; } + else { dwResult = dwErr; break; } + } + char* newBuf = (rBuf==NULL)? + calloc(dwHasRead+dwRead, sizeof(char)) : + realloc(rBuf, (dwHasRead+dwRead)*sizeof(char)); + if (newBuf == NULL) { dwResult = GetLastError(); break; } + rBuf = newBuf; + memcpy(&rBuf[dwHasRead], buf, dwRead); + dwHasRead += dwRead; + } + + if (dwResult != ERROR_SUCCESS) { + if (dwHasRead > 0 && rBuf != NULL) { free(rBuf); } + } else { + arg->buf = rBuf; + arg->len = dwHasRead; + } + CloseHandle(hndl); + return dwResult; } -winThreadRead(Out) -winThreadRead(Err) - -#undef winThreadRead - static DWORD winCmd(char* arg, u64 iLen, char* iBuf, DWORD* code, @@ -131,7 +136,7 @@ static DWORD winCmd(char* arg, si.hStdInput = hInpR; si.hStdOutput = hOutW; si.hStdError = hErrW; - si.dwFlags |= STARTF_USESTDHANDLES | STARTF_UNTRUSTEDSOURCE; + si.dwFlags |= STARTF_USESTDHANDLES; PROCESS_INFORMATION pi; ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); @@ -145,18 +150,16 @@ static DWORD winCmd(char* arg, CloseHandle(hOutW); CloseHandle(hErrW); - // Now we are dealing with the globals and threads - - - - dataInp = iBuf; - sizeInp = iLen; + // Spawn the thread to deal with redirected io + ThreadIO data0 = {hInpW, iBuf, iLen}; + ThreadIO data1 = {hOutR, NULL, 0}; + ThreadIO data2 = {hErrR, NULL, 0}; DWORD exitCode = -1; HANDLE lpThreads[3]; - lpThreads[0] = CreateThread(NULL, 0, winThreadWriteIn, (LPVOID)hInpW, 0, NULL); - lpThreads[1] = CreateThread(NULL, 0, winThreadReadOut, (LPVOID)hOutR, 0, NULL); - lpThreads[2] = CreateThread(NULL, 0, winThreadReadErr, (LPVOID)hErrR, 0, NULL); + lpThreads[0] = CreateThread(NULL, 0, winThreadWrite, (LPVOID)&data0, 0, NULL); + lpThreads[1] = CreateThread(NULL, 0, winThreadRead, (LPVOID)&data1, 0, NULL); + lpThreads[2] = CreateThread(NULL, 0, winThreadRead, (LPVOID)&data2, 0, NULL); for (int i = 0; i < 3; ++i) { if (lpThreads[i] == NULL) { dwResult = GetLastError(); goto error; } @@ -180,8 +183,8 @@ static DWORD winCmd(char* arg, // Give outputs *code = exitCode; - *oLen = sizeOut; *oBuf = dataOut; - *eLen = sizeErr; *eBuf = dataErr; + *oLen = data1.len; *oBuf = data1.buf; + *eLen = data2.len; *eBuf = data2.buf; error: // Close handles @@ -189,7 +192,5 @@ error: CloseHandle(pi.hProcess); CloseHandle(pi.hThread); - - return dwResult; } From 893ef857d7b24eb76d266a86be9f5a2bc9ac10dd Mon Sep 17 00:00:00 2001 From: vylsaz Date: Sun, 4 Feb 2024 02:45:25 +0000 Subject: [PATCH 3/3] Add comments --- src/utils/file.c | 2 ++ src/windows/winError.c | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index c668b4f5..ae7fb164 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -318,6 +318,8 @@ B mmap_file(B path) { thrM("failed to mmap file"); } #else + // see https://learn.microsoft.com/en-us/windows/win32/memory/creating-a-view-within-a-file + HANDLE hFile = CreateFileA( p, GENERIC_READ, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); diff --git a/src/windows/winError.c b/src/windows/winError.c index 67d2ca41..e89fb429 100644 --- a/src/windows/winError.c +++ b/src/windows/winError.c @@ -1,11 +1,13 @@ #include +// memory is allocated and should be freed with LocalFree() +// for now this memory is leaked static char* winErrorEx(DWORD dwError) { char* buffer = NULL; - DWORD dwFlags = FORMAT_MESSAGE_MAX_WIDTH_MASK - | FORMAT_MESSAGE_ALLOCATE_BUFFER - | FORMAT_MESSAGE_FROM_SYSTEM - | FORMAT_MESSAGE_IGNORE_INSERTS; + DWORD dwFlags = FORMAT_MESSAGE_MAX_WIDTH_MASK // no newlines + | FORMAT_MESSAGE_ALLOCATE_BUFFER // allocate memory + | FORMAT_MESSAGE_FROM_SYSTEM // get error message + | FORMAT_MESSAGE_IGNORE_INSERTS; // no argument (NULL) DWORD dwResult = FormatMessageA(dwFlags, NULL, dwError, 0, (LPSTR)&buffer, 0, NULL); if (dwResult==0 || buffer==NULL) { fatal("Failed to get error message from FormatMessageA()");