Re: What's left?

Tom Lane wrote:
> Claudio Natoli <claudio.natoli@memetrics.com> writes:
> > One important thing I forgot, that someone could start looking at now:
> >  *  backends keeping files open when other backends are trying to
> > delete/rename them
>
> > We must do better for the official port,
>
> Why?  The procedure you mentioned seems perfectly adequate to me,
> seeing that it's a bit of a corner case to start with.

I don't see this as a corner case, except it being a corner case
operating system.  :-)

I think it will very likely rename/unlink will fail because of the file
descriptor cache kept by each backend.

I am attaching dir.c from the PeerDirect port.  It handles unlink
failures by appending the failed file name to a file that is later read
and another unlink attempted.  Perhaps this is something we can do, and
have try unlinks after each checkpoint.

PeerDirect handles rename by just looping.  We really can't delay a
rename.  There is discussion in the Win32 TODO detail that goes over
some options, I think.

--
  Bruce Momjian                        |  http://candle.pha.pa.us
  pgman@candle.pha.pa.us               |  (610) 359-1001
  +  If your life is a hard drive,     |  13 Roberts Road
  +  Christ can be your backup.        |  Newtown Square, Pennsylvania 19073
#include "port/win.h"

#define WIN32_LEAN_AND_MEAN        /* Exclude rarely-used stuff from Windows headers */
#include "windows.h"

#include <stdlib.h>
#include <stdio.h>

#ifdef _LIB
#include "postgres.h"
#include "miscadmin.h"
#endif

int
win32_rmdir(const char *dirname)
{
    BOOL ans = RemoveDirectory(dirname);

    if (!ans && GetLastError() == ERROR_DIR_NOT_EMPTY) {
        /* Blow away the contents recursively and try again */
        WIN32_FIND_DATA        dir_find_data;
        char buffer[MAX_PATH*2 + 1];
        HANDLE find_handle;

        strcpy(buffer, dirname);
        strcat(buffer, "\\*");

        ans = TRUE;
        find_handle = FindFirstFile(buffer, &dir_find_data);

        while (find_handle && find_handle != INVALID_HANDLE_VALUE) {
            if (strcmp(dir_find_data.cFileName, ".") != 0 && strcmp(dir_find_data.cFileName, "..") != 0) {
                strcpy(buffer + strlen(dirname) + 1, dir_find_data.cFileName);

                if (dir_find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
                    ans = (win32_rmdir(buffer) == 0);
                else
                    ans = DeleteFile(buffer);
            }

            if (!ans || !FindNextFile(find_handle, &dir_find_data))
                break;
        }

        FindClose(find_handle);
        ans = ans && RemoveDirectory(dirname);
    }

    return (ans ? 0 : -1);
}

/* Recursively copy srcdir to destdir.  Destdir is created
 * if needed.  If destdir exists already but is not a
 * directory, it is an error.  Returns 0 for success,
 * -1 otherwise.
 */
int copydir(const char* srcdir, const char* destdir)
{
    WIN32_FIND_DATA dir_find_data;
    char src_buffer[MAX_PATH*2 + 1];
    char dest_buffer[MAX_PATH*2 + 1];
    HANDLE find_handle = FindFirstFile(destdir, &dir_find_data);
    BOOL ans;

    if (find_handle && (find_handle != INVALID_HANDLE_VALUE)) {
        /* destdir exists, now make sure it is a directory */
        if (dir_find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
            ans = 1;
        else
            ans = 0;
        FindClose(find_handle);
    }
    else {
        /* Create the destdir */
        ans = CreateDirectory(destdir, NULL);
    }
    FindClose(find_handle);

    if (!ans)
        return -1;

    /* Get ready to do some copying */
    strcpy(src_buffer, srcdir);
    strcat(src_buffer, "\\*");

    strcpy(dest_buffer, destdir);
    strcat(dest_buffer, "\\");

    find_handle = FindFirstFile(src_buffer, &dir_find_data);
    if (find_handle && find_handle != INVALID_HANDLE_VALUE)
    {
        do
        {
            if (strcmp(dir_find_data.cFileName, ".") == 0)
                continue;
            if (strcmp(dir_find_data.cFileName, "..") == 0)
                continue;

            strcpy(src_buffer + strlen(srcdir) + 1, dir_find_data.cFileName);
            strcpy(dest_buffer + strlen(destdir) + 1, dir_find_data.cFileName);

            if (dir_find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
                ans = (copydir(src_buffer, dest_buffer) == 0);
            else
                ans = CopyFile(src_buffer, dest_buffer, FALSE);

            if (!ans)
                break;
        }
        while (FindNextFile(find_handle, &dir_find_data));
        FindClose(find_handle);
    }

    return (ans ? 0 : -1);
}

#define VALID_FLAG    0xcade

typedef struct {
    DWORD            valid_flag;
    char            dir_name[MAX_PATH + 1];
    dirent            *dir_info;
    HANDLE            search_handle;
} pg_win32_dir_struct;

DIR* opendir(const char* name)
{
    WIN32_FIND_DATA    dir_find_data;
    HANDLE file_handle = FindFirstFile(name, &dir_find_data);
    pg_win32_dir_struct* the_dir = NULL;

    if (file_handle && (file_handle != INVALID_HANDLE_VALUE))
    {
        FindClose(file_handle);
        if (dir_find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
        {
            /* Found a decent directory -- return it */
            the_dir = (pg_win32_dir_struct*)malloc(sizeof(pg_win32_dir_struct));
            the_dir->valid_flag = VALID_FLAG;
            strcpy(the_dir->dir_name, name);
            strcat(the_dir->dir_name, "/");
            the_dir->dir_info = (dirent*)malloc(sizeof(dirent));
            the_dir->search_handle = NULL;
        }
    }

    return the_dir;
}

dirent* readdir(DIR* dir)
{
    WIN32_FIND_DATA        find_data;
    static dirent s_dirent;

    pg_win32_dir_struct* the_dir = (pg_win32_dir_struct*)dir;

    if (!the_dir || the_dir->valid_flag != VALID_FLAG)
        /* Bad user -- gave invalid DIR */
        return NULL;

    if (the_dir->search_handle == NULL) {
        /* This is the first call */
        char buffer[MAX_PATH+1];
        HANDLE file_handle;

        strcpy(buffer, the_dir->dir_name);
        strcat(buffer, "*");

        file_handle = FindFirstFile(buffer, &find_data);

        if (file_handle
            && (file_handle != INVALID_HANDLE_VALUE)) {
            the_dir->search_handle = file_handle;
        }
    }
    else if (!FindNextFile(the_dir->search_handle, &find_data)) {
        FindClose(the_dir->search_handle);
        the_dir->search_handle = NULL;
    }

    if (the_dir->search_handle) {
        strcpy(the_dir->dir_info->d_name, find_data.cFileName);
        the_dir->dir_info->d_type = find_data.dwFileAttributes;
        return the_dir->dir_info;
    }
    else
        return NULL;
}

int closedir(DIR* dir)
{
    pg_win32_dir_struct* the_dir = (pg_win32_dir_struct*)dir;

    if (the_dir && the_dir->valid_flag == VALID_FLAG) {
        the_dir->valid_flag = -1;

        if (the_dir->search_handle)
            FindClose(the_dir->search_handle);
        free(the_dir->dir_info);
        the_dir->dir_info = ((dirent*)1);
        free(the_dir);
    }

    return 0;
}

#ifdef _LIB
#define UNLINK_LATER_FILE "pg_win32_unlink_later.data"

BOOL win32_save_unlink_for_later(char* filename)
{
    FILE* later_file;
    char buffer[1024];
    int count = 0;

    sprintf(buffer, "%s/global/%s", DataDir, UNLINK_LATER_FILE);
    later_file = fopen(buffer, "a+");

    while (!later_file && count++ < 5) {
        Sleep(100);
        later_file = fopen(buffer, "a+");
    }

    if (later_file) {
        SYSTEMTIME sys_time;
        GetLocalTime(&sys_time);
        fprintf(later_file, "%02hu/%02hu/%04hu %02hu:%02hu:%02hu\t"
                            "%s\n",
                                sys_time.wMonth, sys_time.wDay, sys_time.wYear,
                                sys_time.wHour, sys_time.wMinute, sys_time.wSecond,
                                filename);
        fclose(later_file);
        elog(NOTICE, "Could not delete %s -- will try again later", filename);
        return true;
    }
    else
        return false;
}

void win32_unlink_leftover_files()
{
    FILE* later_file;
    char later_file_path[1024];

    sprintf(later_file_path, "%s/global/%s", DataDir, UNLINK_LATER_FILE);
    later_file = fopen(later_file_path, "r");

    if (later_file) {
        char temp_file_path[1024], filename[1024];
        FILE* temp_file = NULL;
        SYSTEMTIME orig_time;
        memset(&orig_time, 0, sizeof(SYSTEMTIME));

        while (fscanf(later_file, "%02hu/%02hu/%04hu %02hu:%02hu:%02hu\t%s\n",
                                    &orig_time.wMonth, &orig_time.wDay, &orig_time.wYear,
                                    &orig_time.wHour, &orig_time.wMinute, &orig_time.wSecond,
                                    filename) != EOF)
        {
            if (DeleteFile(filename))
                elog(DEBUG, "Successfully deleted file %s, which had been postponed from an earlier attempt",
filename);
            else
                if (GetLastError() != ERROR_FILE_NOT_FOUND) {
                    if (!temp_file) {
                        sprintf(temp_file_path, "%s/global/%s.%d", DataDir, UNLINK_LATER_FILE, getpid());
                        temp_file = fopen(temp_file_path, "w");
                    }

                    if (temp_file) {
                        fprintf(temp_file, "%02hu/%02hu/%04hu %02hu:%02hu:%02hu\t"
                                            "%s\n",
                                orig_time.wMonth, orig_time.wDay, orig_time.wYear,
                                orig_time.wHour, orig_time.wMinute, orig_time.wSecond,
                                filename);
                    }
                    else {
                        elog(DEBUG, "Could not delete file %s.  Please delete by hand.", filename);
                    }
                }
        }

        fclose(later_file);
        DeleteFile(later_file_path);

        if (temp_file) {
            fclose(temp_file);
            rename(temp_file_path, later_file_path);
        }
    }
}
#endif    /* _LIB */