Merge pull request #131 from vylsaz/utf16

Switch to wide character functions for file handling on Windows
This commit is contained in:
dzaima 2025-01-16 22:08:19 +02:00 committed by GitHub
commit 64a3f44f3f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 197 additions and 129 deletions

View File

@ -1153,32 +1153,44 @@ static i32 sh_core(bool raw, B x, usz xia, B inObj, u64 iLen, B* s_outp, B* s_er
}
#elif defined(_WIN32) || defined(_WIN64)
#define HAS_SH 1
#include "../windows/winError.c"
#include "../windows/sh.c"
#include "../windows/winError.h"
#include "../windows/utf16.h"
#include "../windows/sh.h"
static i32 sh_core(bool raw, B x, usz xia, B inObj, u64 iLen, B* s_outp, B* s_errp) {
// allocate args
u64 arglen = 0;
TSALLOC(WCHAR, arg, 8);
SGetU(x)
for (u64 i = 0; i < xia; i++) {
B c = GetU(x, i);
if (isAtm(c) || RNK(c)!=1) thrM("•SH: 𝕩 must be a list of strings");
u64 len = utf8lenB(c);
arglen += 1+2+2*len;
// space or 0, quotes, worst-case scenario (every character needs escaping)
}
TALLOC(char, arg, arglen);
char* pos = arg;
for (u64 i = 0; i < xia; i++) {
B c = GetU(x, i);
u64 len = utf8lenB(c);
TALLOC(char, cstr, len+1);
toUTF8(c, cstr);
cstr[len] = 0;
pos = winQuoteCmdArg(len, cstr, pos);
*(pos++) = (xia==i+1)? '\0' : ' ';
TFREE(cstr)
assert(pos <= arg+arglen);
u64 len = utf16lenB(c);
TALLOC(WCHAR, wstr, len);
toUTF16(c, wstr);
// https://learn.microsoft.com/en-gb/archive/blogs/twistylittlepassagesallalike/everyone-quotes-command-line-arguments-the-wrong-way
u64 backslashes = 0;
bool quote = len==0 || NULL!=wcspbrk(wstr, L" \t\n\v\"");
if (quote) { TSADD(arg, L'\"'); }
for (u64 j = 0; j < len; ++j) {
WCHAR x = wstr[j];
if (x==L'\\') {
backslashes += 1;
} else {
if (x==L'\"') {
for (u64 k = 0; k < 1+backslashes; ++k) { TSADD(arg, L'\\'); }
}
backslashes = 0;
}
TSADD(arg, x);
}
if (quote) {
for (u64 k = 0; k < backslashes; ++k) { TSADD(arg, L'\\'); }
TSADD(arg, L'\"');
}
TSADD(arg, (xia==i+1)? L'\0' : L' ');
TFREE(wstr);
}
// allocate stdin
@ -1200,7 +1212,7 @@ static i32 sh_core(bool raw, B x, usz xia, B inObj, u64 iLen, B* s_outp, B* s_er
u64 eLen = 0; char* eBuf;
DWORD dwResult = winCmd(arg, iLen, iBuf, &code, &oLen, &oBuf, &eLen, &eBuf);
if (iLen>0) { if (raw) free_chars(iBufRaw); else TFREE(iBuf); } // FREE_INPUT
TFREE(arg)
TSFREE(arg);
if (dwResult != ERROR_SUCCESS) {
thrF("•SH: Failed to run command: %S", winErrorEx(dwResult));
}
@ -1208,8 +1220,8 @@ static i32 sh_core(bool raw, B x, usz xia, B inObj, u64 iLen, B* s_outp, B* s_er
// prepare output
u8* op; *s_outp = m_c8arrv(&op, oLen);
u8* ep; *s_errp = m_c8arrv(&ep, eLen);
if (oLen > 0 && oBuf != NULL) memcpy(op, oBuf, oLen*sizeof(char)); free(oBuf);
if (eLen > 0 && eBuf != NULL) memcpy(ep, eBuf, eLen*sizeof(char)); free(eBuf);
if (oBuf!=NULL) { memcpy(op, oBuf, oLen*sizeof(char)); free(oBuf); }
if (eBuf!=NULL) { memcpy(ep, eBuf, eLen*sizeof(char)); free(eBuf); }
return (i32)code;
}
#else

View File

@ -6,26 +6,57 @@
#include <unistd.h>
#include <errno.h>
#if defined(_WIN32) || defined(_WIN64)
#if defined(_WIN32)
#include <direct.h>
#include "../windows/utf16.h"
// use wide char functions for unicode support / longer paths (potentially)
#include "../windows/realpath.c"
#endif
#if !defined(_WIN32)
typedef char* OsStr;
#define OS_C(C) C
#define toOsStr toCStr
#define freeOsStr freeCStr
#define OsStrDecode0 utf8Decode0
#else
typedef WCHAR* OsStr;
#define OS_C(C) L##C
#define toOsStr toWStr
#define freeOsStr freeWStr
#define OsStrDecode0 utf16Decode0
#define DIR _WDIR
#define dirent _wdirent
#define opendir _wopendir
#define readdir _wreaddir
#define closedir _wclosedir
#define mkdir(P, IGNORE) _wmkdir(P)
#define access _waccess
#define rename _wrename
#define unlink _wunlink
#endif
FILE* file_open(B path, char* desc, char* mode) { // doesn't consume
#if !defined(_WIN32)
char* p = toCStr(path);
FILE* f = fopen(p, mode);
freeCStr(p);
#else
WCHAR wmode[8] = {0};
u64 len = strlen(mode);
assert(len<(sizeof(wmode)/sizeof(WCHAR)));
for (u64 i = 0; i<len; ++i) wmode[i] = (WCHAR)mode[i];
WCHAR *p = toWStr(path);
FILE* f = _wfopen(p, wmode);
freeWStr(p);
#endif
if (f==NULL) thrF("Couldn't %S file \"%R\"", desc, path);
return f;
}
static DIR* dir_open(B path) { // doesn't consume
u64 plen = utf8lenB(path);
TALLOC(char, p, plen+1);
toUTF8(path, p);
p[plen] = 0;
OsStr p = toOsStr(path);
DIR* f = opendir(p);
TFREE(p);
freeOsStr(p);
if (f==NULL) thrF("Couldn't open directory \"%R\"", path);
return f;
}
@ -171,16 +202,13 @@ B path_abs(B path) {
return path; // lazy
#else
if (q_N(path)) return path;
u64 plen = utf8lenB(path);
TALLOC(char, p, plen+1);
toUTF8(path, p);
p[plen] = 0;
char* res = realpath(p, NULL);
OsStr p = toOsStr(path);
OsStr res = realpath(p, NULL);
if (res==NULL) thrF("Failed to resolve \"%R\": %S", path, strerror(errno));
B r = utf8Decode0(res);
B r = OsStrDecode0(res);
free(res);
dec(path);
TFREE(p);
freeOsStr(p);
return r;
#endif
}
@ -251,8 +279,10 @@ B path_list(B path) {
struct dirent *c;
B res = emptySVec();
while ((c = readdir(d)) != NULL) {
char* name = c->d_name;
if (name[0]=='.'? !(name[1]==0 || (name[1]=='.'&&name[2]==0)) : true) res = vec_addN(res, utf8Decode(name, strlen(name)));
OsStr name = c->d_name;
if (name[0]==OS_C('.')? !(name[1]==0 || (name[1]==OS_C('.')&&name[2]==0)) : true) {
res = vec_addN(res, OsStrDecode0(name));
}
}
closedir(d);
dec(path);
@ -267,7 +297,7 @@ B path_list(B path) {
#include <unistd.h>
#else
#include <windows.h>
#include "../windows/winError.c"
#include "../windows/winError.h"
#endif
typedef struct MmapHolder {
@ -301,9 +331,9 @@ static NOINLINE Arr* mmapH_slice(B x, usz s, usz ia) {
}
B mmap_file(B path) {
#if !defined(_WIN32)
char* p = toCStr(path);
dec(path);
#if !defined(_WIN32)
int fd = open(p, 0);
freeCStr(p);
if (fd==-1) thrF("Failed to open file: %S", strerror(errno));
@ -316,11 +346,12 @@ B mmap_file(B path) {
}
#else
// see https://learn.microsoft.com/en-us/windows/win32/memory/creating-a-view-within-a-file
HANDLE hFile = CreateFileA(
WCHAR* p = toWStr(path);
dec(path);
HANDLE hFile = CreateFileW(
p, GENERIC_READ, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, NULL,
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
freeCStr(p);
freeWStr(p);
if (hFile==INVALID_HANDLE_VALUE) thrF("Failed to open file: %S", winError());
LARGE_INTEGER fileSize;
if (!GetFileSizeEx(hFile, &fileSize)) {
@ -329,7 +360,7 @@ B mmap_file(B path) {
}
u64 len = fileSize.QuadPart;
HANDLE hMapFile = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
HANDLE hMapFile = CreateFileMappingW(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
if (hMapFile==INVALID_HANDLE_VALUE) {
CloseHandle(hFile);
thrF("Failed to create file mapping: %S", winError());
@ -341,7 +372,7 @@ B mmap_file(B path) {
thrF("Failed to map view of file: %S", winError());
}
#endif
MmapHolder* holder = m_arrUnchecked(sizeof(MmapHolder), t_mmapH, len);
holder->a = data;
#if !defined(_WIN32)
@ -379,39 +410,39 @@ void mmap_init() { }
#include <sys/stat.h>
bool dir_create(B path) {
char* p = toCStr(path);
#if defined(_WIN32) || defined(_WIN64)
bool r = _mkdir(p) == 0;
#else
bool r = mkdir(p, S_IRWXU) == 0;
#endif
freeCStr(p);
OsStr p = toOsStr(path);
bool r = mkdir(p, S_IRWXU) == 0;
freeOsStr(p);
return r;
}
bool path_rename(B old_path, B new_path) {
char* old = toCStr(old_path);
char* new = toCStr(new_path);
OsStr old = toOsStr(old_path);
OsStr new = toOsStr(new_path);
// TODO Fix race condition, e.g., with renameat2 on Linux, etc.
bool ok = access(new, F_OK) != 0 && rename(old, new) == 0;
freeCStr(new);
freeCStr(old);
freeOsStr(new);
freeOsStr(old);
dec(old_path);
return ok;
}
bool path_remove(B path) {
char* p = toCStr(path);
OsStr p = toOsStr(path);
bool ok = unlink(p) == 0;
freeCStr(p);
freeOsStr(p);
dec(path);
return ok;
}
int path_stat(struct stat* s, B path) { // doesn't consume; get stat of s; errors if path isn't string; returns non-zero on failure
char* p = toCStr(path);
OsStr p = toOsStr(path);
#if !defined(_WIN32)
int r = stat(p, s);
freeCStr(p);
#else
int r = wstat(p, s);
#endif
freeOsStr(p);
return r;
}

View File

@ -1,9 +1,6 @@
#include "realpath.h"
char* realpath (const char *__restrict path, char *__restrict resolved_path) {
return _fullpath(NULL, path, 0);
WCHAR* realpath (const WCHAR*__restrict path, WCHAR*__restrict resolved_path) {
return _wfullpath(NULL, path, 0);
}
bool winIsAbsolute(const char* path) { // TODO something more proper
return *path && path[1]==':' && (!path[2] || path[2]=='/' || path[2]=='\\');
}
}

View File

@ -1,7 +0,0 @@
#ifndef REALPATH_H
#define REALPATH_H
char* realpath(const char *__restrict path, char *__restrict resolved_path);
bool winIsAbsolute(const char* path);
#endif /* REALPATH_H */

View File

@ -1,49 +1,5 @@
#include <windows.h>
// https://github.com/libuv/libuv/blob/v1.23.0/src/win/process.c#L454-L524
static char* winQuoteCmdArg(u64 len, char* source, char* target) {
if (len == 0) {
// Need double quotation for empty argument
*(target++) = '"';
*(target++) = '"';
return target;
}
if (NULL == strpbrk(source, " \t\"")) {
// No quotation needed
memcpy(target, source, len * sizeof(char)); target += len;
return target;
}
if (NULL == strpbrk(source, "\"\\")) {
// No embedded double quotes or backlashes, so I can just wrap
// quote marks around the whole thing.
*(target++) = '"';
memcpy(target, source, len * sizeof(char)); target += len;
*(target++) = '"';
return target;
}
*(target++) = '"';
char *start = target;
int quote_hit = 1;
for (u64 i = 0; i < len; ++i) {
*(target++) = source[len - 1 - i];
if (quote_hit && source[len - 1 - i] == '\\') {
*(target++) = '\\';
} else if (source[len - 1 - i] == '"') {
quote_hit = 1;
*(target++) = '\\';
} else {
quote_hit = 0;
}
}
target[0] = '\0'; _strrev(start);
*(target++) = '"';
return target;
}
typedef struct {
HANDLE hndl;
char* buf;
@ -72,20 +28,19 @@ static DWORD WINAPI winThreadRead(LPVOID arg0) {
DWORD dwResult = ERROR_SUCCESS;
ThreadIO* arg = arg0;
HANDLE hndl = arg->hndl;
u8 buf[1024] = {0};
u8 buf[4096] = {0};
const usz bufSize = sizeof(buf)/sizeof(u8);
DWORD dwRead = 0, dwHasRead = 0;
char* rBuf = NULL;
for (;;) {
ZeroMemory(buf, bufSize);
BOOL bOk = ReadFile(hndl, buf, bufSize, &dwRead, NULL);
if (dwRead == 0) { break; }
if (!bOk) {
DWORD dwErr = GetLastError();
if (dwErr == ERROR_BROKEN_PIPE) { break; }
else { dwResult = dwErr; break; }
dwResult = GetLastError();
break;
}
char* newBuf = (rBuf==NULL)?
char* newBuf = (rBuf == NULL)?
calloc(dwHasRead+dwRead, sizeof(char)) :
realloc(rBuf, (dwHasRead+dwRead)*sizeof(char));
if (newBuf == NULL) { dwResult = GetLastError(); break; }
@ -95,7 +50,7 @@ static DWORD WINAPI winThreadRead(LPVOID arg0) {
}
if (dwResult != ERROR_SUCCESS) {
if (dwHasRead > 0 && rBuf != NULL) { free(rBuf); }
if (rBuf != NULL) { free(rBuf); }
} else {
arg->buf = rBuf;
arg->len = dwHasRead;
@ -104,7 +59,7 @@ static DWORD WINAPI winThreadRead(LPVOID arg0) {
return dwResult;
}
static DWORD winCmd(char* arg,
static DWORD winCmd(WCHAR* arg,
u64 iLen, char* iBuf,
DWORD* code,
u64* oLen, char** oBuf,
@ -117,7 +72,7 @@ static DWORD winCmd(char* arg,
// Create pipes
SECURITY_ATTRIBUTES sa;
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
sa.nLength = sizeof(sa);
sa.lpSecurityDescriptor = NULL;
sa.bInheritHandle = TRUE;
@ -130,9 +85,9 @@ static DWORD winCmd(char* arg,
SetHandleInformation(hErrR, HANDLE_FLAG_INHERIT, 0);
// Set up
STARTUPINFO si;
ZeroMemory(&si, sizeof(STARTUPINFO));
si.cb = sizeof(STARTUPINFO);
STARTUPINFOW si;
ZeroMemory(&si, sizeof(si));
si.cb = sizeof(si);
si.hStdInput = hInpR;
si.hStdOutput = hOutW;
si.hStdError = hErrW;
@ -142,7 +97,7 @@ static DWORD winCmd(char* arg,
ZeroMemory(&pi, sizeof(PROCESS_INFORMATION));
// Create the child process
BOOL bSuccess = CreateProcessA(NULL, arg, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi);
BOOL bSuccess = CreateProcessW(NULL, arg, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi);
if (!bSuccess) { return GetLastError(); }
// Close the unneeded handles

80
src/windows/utf16.h Normal file
View File

@ -0,0 +1,80 @@
#include "../utils/talloc.h"
#include <windows.h>
static u64 utf16lenB(B x) { // doesn't consume
assert(isArr(x));
SGetU(x)
usz ia = IA(x);
u64 res = 0;
for (usz i = 0; i < ia; ++i) {
u32 c = o2c(GetU(x,i));
res+= 1+(c > 0xFFFF);
}
return res;
}
FORCE_INLINE void utf16_w(WCHAR** buf_i, u32 c)
{
WCHAR* buf = *buf_i;
if (c<=0xFFFF) {
*buf++ = c;
} else {
assert(c <= 0x10FFFF);
*buf++ = ((c-0x10000) >> 10)+0xD800;
*buf++ = ((c-0x10000)&0x3FF)+0xDC00;
}
*buf_i = buf;
}
static void toUTF16(B x, WCHAR* p) {
SGetU(x)
usz ia = IA(x);
for (u64 i = 0; i < ia; ++i) utf16_w(&p, o2cG(GetU(x,i)));
}
static B utf16Decode(const WCHAR* s, i64 len) {
#define UTF16_MASK 0xFC00
#define UTF16_IS_HI(WC) ((UTF16_MASK&(WC))==0xD800) /* 0xD800..0xDBFF */
#define UTF16_IS_LO(WC) ((UTF16_MASK&(WC))==0xDC00) /* 0xDC00..0xDFFF */
#define UTF16_SURROGATE(HI, LO) (0x10000+(((HI)-0xD800) << 10)+((LO)-0xDC00))
u64 sz = 0;
for (i64 j = 0; ; ++j) {
if (j>=len) {
if (j!=len) assert(0);
break;
}
if (UTF16_IS_HI(s[j]) && j+1<len && UTF16_IS_LO(s[j+1])) ++j;
++sz;
}
u32* rp; B r = m_c32arrv(&rp, sz);
u64 p = 0;
for (i64 i = 0; i < len; ++i) {
if (UTF16_IS_HI(s[i]) && i+1<len && UTF16_IS_LO(s[i+1])) {
rp[p++] = UTF16_SURROGATE(s[i], s[i+1]);
++i;
} else {
rp[p++] = s[i];
}
}
assert(p==sz);
return r;
#undef UTF16_MASK
#undef UTF16_IS_HI
#undef UTF16_IS_LO
#undef UTF16_SURROGATE
}
static B utf16Decode0(const WCHAR* s) {
return utf16Decode(s, wcslen(s));
}
static WCHAR* toWStr(B x) { // doesn't consume
u64 len = utf16lenB(x);
TALLOC(WCHAR, p, len+1);
toUTF16(x, p);
p[len] = 0;
return p;
}
static void freeWStr(WCHAR* p) {
TFREE(p);
}