Code Snippets for a short article or series on error recovery models and reliability

readme.md

This article covers methods for improving reliability of a complex component.

We start with two methods that rely on crashing less to improve reliability.

• Two Phase Commit
• Hardening and Stress

We move on to a method which favors crashing early with the maximum information available to aid in correcting the issue.

• Fail Fast

error_recovery.cpp

typedef unsigned long ErrorCode;
typedef void* MemoryHandle;

ErrorCode OutOfMemory = 0x1;
ErrorCode Success = 0x2;
ErrorCode TerribleFailure = 0x3;

void FreeHandle(MemoryHandle handle)
{
}

class MemoryObject
{
public:
    ErrorCode Init()
    {
        return Success;
    }
};

MemoryHandle MemoryPropagator()
{
    return new MemoryObject();
}

ErrorCode MemoryAndErrorPropagator(MemoryHandle* handle)
{
    ErrorCode code = Success;
    MemoryObject* object = new MemoryObject();
    if (handle != nullptr)
    {
        code = object->Init();
        if (code == Success)
        {
            *handle = object;
        }
        else
        {
            delete object;
        }
    }
    else
    {
        code = OutOfMemory;
    }
    return code;
}

ErrorCode ErrorRecoveryInitiator(MemoryHandle handle)
{
    return TerribleFailure;
}

ErrorCode ErrorRecoveryPropagator()
{
    ErrorCode code = Success;
    MemoryHandle handle = MemoryPropagator();

    // First type of robustness is convert allocation failures to our error model, this is used when integrating with code outside
    // of your error model that communicates to you through allocation failures by return nullptr.
    if (handle != nullptr)
    {
        MemoryHandle handle2;
        code = MemoryAndErrorPropagator(&handle2);
        if (code == Success)
        {
            code = ErrorRecoveryInitiator(handle2);
            if (code != Success)
            {
                // We are now two calls deep and must ALSO clean up handle2
                FreeHandle(handle2);
            }
        }

        // We have to do something with the original handle on failure. Perhaps it would have been owned by the second handle, but now it can't be
        // since we failed to allocate or potentially failed to init the second handle.
        if (code != Success)
        {
            FreeHandle(handle);
        }
    }
    else
    {
        code = OutOfMemory;
    }
    return code;
}

fail_fast.cpp

typedef void* MemoryHandle;

__declspec(noreturn)
void InduceAbandonment()
{
    // RaiseFailFastException - https://msdn.microsoft.com/en-us/library/windows/desktop/dd941688(v=vs.85).aspx
}
__declspec(noreturn)
void InitializationFailure()
{
    InduceAbandonment();
}
__declspec(noreturn)
void TerribleFailure()
{
    InduceAbandonment();
}
template <typename T>
T* MemoryAllocation()
{
    T* obj = new T();
    if (!obj)
    {
        InduceAbandonment();
    }
    return obj;
}

class MemoryObject
{
public:
    void Init()
    {
        InitializationFailure();
    }
};

MemoryHandle MemoryPropagator()
{
    return MemoryAllocation<MemoryObject>();
}

void MemoryAndErrorPropagator(MemoryHandle* handle)
{
    MemoryObject* object = MemoryAllocation<MemoryObject>();
    object->Init();
}

void ErrorRecoveryInitiator(MemoryHandle handle)
{
    TerribleFailure();
}

void ErrorRecoveryPropagator()
{
    // Any failures are now program terminations. So this function cleans up very well.
    MemoryHandle handle = MemoryPropagator();
    MemoryHandle handle2;
    MemoryAndErrorPropagator(&handle2);
    ErrorRecoveryInitiator(handle2);
}