Simple MicroPython ESP32 RMT NeoPixel / WS2812B driver.

pixels.py

# Copyright public licence and also I don't care.
# 2020 Josh "NeverCast" Lloyd.
from micropython import const 
from esp32 import RMT

# The peripheral clock is 80MHz or 12.5 nanoseconds per clock.
# The smallest precision of timing requried for neopixels is
# 0.35us, but I've decided to go with 0.05 microseconds or
# 50 nanoseconds. 50 nanoseconds = 12.5 * 4 clocks.
# By dividing the 80MHz clock by 4 we get a clock every 50 nanoseconds.

# Neopixel timing in RMT clock counts.
T_0H = const(35 // 5) # 0.35 microseconds / 50 nanoseconds
T_1H = const(70 // 5) # 0.70 microseconds / 50 nanoseconds
T_0L = const(80 // 5) # 0.80 microseconds / 50 nanoseconds
T_1L = const(60 // 5) # 0.60 microseconds / 50 nanoseconds

# Encoded timings for bits 0 and 1.
D_ZERO = (T_0H, T_0L)
D_ONE = (T_1H, T_1L)

# [D_ONE if ((channel >> bit) & 1) else D_ZERO for channel in channels for bit in range(num_bits - 1, -1, -1)]
# Reset signal is low for longer than 50 microseconds.
T_RST = const(510 // 5) # > 50 microseconds / 50 nanoseconds

# Channel width in bits 
CHANNEL_WIDTH = const(8)

class Pixels:
    def __init__(self, pin, pixel_count, rmt_channel=1, pixel_channels=3):
        self.rmt = RMT(rmt_channel, pin=pin, clock_div=4)
        # pixels stores the data sent out via RMT
        self.channels = pixel_channels
        single_pixel = (0,) * pixel_channels
        self.pixels = [D_ZERO * (pixel_channels * CHANNEL_WIDTH)] * pixel_count
        # colors is only used for __getitem__
        self.colors = [single_pixel] * pixel_count

    def write(self):
        # The bus should be idle low ( I think... )
        # So we finish low and start high.
        pulses = tuple()
        for pixel in self.pixels:
            pulses += pixel
        pulses = pulses[:-1] + (T_RST,) # The last low should be long.
        self.rmt.write_pulses(pulses, start=1)

    def __setitem__(self, pixel_index, colors):
        self_colors = self.colors
        self_pixels = self.pixels 
        if isinstance(pixel_index, int):
            # pixels[0] = (r, g, b)
            self_colors[pixel_index] = tuple(colors)
            self_pixels[pixel_index] = tuple(clocks for bit in (D_ONE if ((channel >> bit) & 1) else D_ZERO for channel in colors for bit in range(CHANNEL_WIDTH - 1, -1, -1)) for clocks in bit)
        elif isinstance(pixel_index, slice):
            start = 0 if pixel_index.start is None else pixel_index.start
            stop = len(self.pixels) if pixel_index.stop is None else pixel_index.stop
            step = 1 if pixel_index.step is None else pixel_index.step
            # Assume that if the first colors element is an int, then its not a sequence
            # Otherwise always assume its a sequence of colors
            if isinstance(colors[0], int):
                # pixels[:] = (r,g,b)
                for index in range(start, stop, step):
                    self_colors[index] = tuple(colors)
                    self_pixels[index] = tuple(clocks for bit in (D_ONE if ((channel >> bit) & 1) else D_ZERO for channel in colors for bit in range(CHANNEL_WIDTH - 1, -1, -1)) for clocks in bit)
            else:
                # pixels[:] = [(r,g,b), ...]
                # Assume its a sequence, make it a list so we know the length
                if not isinstance(colors, list):
                    colors = list(colors)
                color_length = len(colors)
                for index in range(start, stop, step):
                    color = colors[(index - start) % color_length]
                    self_colors[index] = tuple(color)
                    self_pixels[index] = tuple(clocks for bit in (D_ONE if ((channel >> bit) & 1) else D_ZERO for channel in color for bit in range(CHANNEL_WIDTH - 1, -1, -1)) for clocks in bit)
        else:
            raise TypeError('Unsupported pixel_index {} ({})'.format(pixel_index, type(pixel_index)))

    def __getitem__(self, pixel_index):
        # slice instances are passed through
        return self.colors[pixel_index]

### All code below this point is test code and can be safely deleted in your own application.

def TEST():
    from machine import Pin
    pin = Pin(18)
    p = Pixels(pin, 5)
    try:
        assert(isinstance(p, Pixels))
        assert(p.channels == 3)
        assert(len(p.colors) == 5)
        assert(len(p.pixels) == 5)
        assert(len(p.colors[0]) == 3)
        assert(sum(p.colors[0]) == 0)
        assert(p[0] == (0, 0, 0))
        old_pixels = p.pixels[0]
        assert(p.pixels[0] == old_pixels)
        p[0] = (1, 2, 3)
        assert(p[0] == (1, 2, 3))
        assert(p.colors[0] == p[0])
        assert(p.pixels[0] != old_pixels)
        assert(p[0:1] == [(1,2,3)])
        assert(p[0:2] == [(1,2,3), (0,0,0)])
        p[0:1] = (1, 3, 5)
        assert(p.colors[0] == (1, 3, 5))
        assert(p.colors[1] == (0, 0, 0))
        assert(p.colors[0:2] == [(1,3,5), (0,0,0)])
        p[0:4] = [(1,1,1), (2,2,2)]
        assert(p.colors[0:4] == [(1,1,1), (2,2,2)]*2)
    finally:
        p.rmt.deinit()
        
    p = Pixels(pin, 5, pixel_channels=4)
    try:
        assert(isinstance(p, Pixels))
        assert(p.channels == 4)
        assert(len(p.colors) == 5)
        assert(len(p.pixels) == 5)
        assert(len(p.colors[0]) == 4)
        assert(sum(p.colors[0]) == 0)
        assert(p[0] == (0, 0, 0, 0))
        old_pixels = p.pixels[0]
        assert(p.pixels[0] == old_pixels)
        p[0] = (1, 2, 3, 4)
        assert(p[0] == (1, 2, 3, 4))
        assert(p.colors[0] == p[0])
        assert(p.pixels[0] != old_pixels)
        assert(p[0:1] == [(1,2,3,4)])
        assert(p[0:2] == [(1,2,3,4), (0,0,0,0)])
        p[0:1] = (1, 3, 5, 7)
        assert(p.colors[0] == (1, 3, 5, 7))
        assert(p.colors[1] == (0, 0, 0, 0))
        assert(p.colors[0:2] == [(1,3,5,7), (0,0,0,0)])
        p[0:4] = [(1,1,1,1), (2,2,2,2)]
        assert(p.colors[0:4] == [(1,1,1,1), (2,2,2,2)]*2)
    finally:
        p.rmt.deinit()

def RAINBOW():
    from machine import Pin
    from time import ticks_ms, ticks_diff
    last = ticks_ms()
    def delta(title=None):
        nonlocal last
        if title is not None:
            print(title,ticks_diff(ticks_ms(), last),'ms')
        last = ticks_ms()
    p = Pin(15)
    pix = Pixels(p, 60)
    rainbow = [[126 , 1 , 0],[114 , 13 , 0],[102 , 25 , 0],[90 , 37 , 0],[78 , 49 , 0],[66 , 61 , 0],[54 , 73 , 0],[42 , 85 , 0],[30 , 97 , 0],[18 , 109 , 0],[6 , 121 , 0],[0 , 122 , 5],[0 , 110 , 17],[0 , 98 , 29],[0 , 86 , 41],[0 , 74 , 53],[0 , 62 , 65],[0 , 50 , 77],[0 , 38 , 89],[0 , 26 , 101],[0 , 14 , 113],[0 , 2 , 125],[9 , 0 , 118],[21 , 0 , 106],[33 , 0 , 94],[45 , 0 , 82],[57 , 0 , 70],[69 , 0 , 58],[81 , 0 , 46],[93 , 0 , 34],[105 , 0 , 22],[117 , 0 , 10]]
    while True:
        rainbow = rainbow[-1:] + rainbow[:-1]
        pix[:] = rainbow
        delta('update')
        pix.write()
        delta('write')

Do you know what part number your strips are? I intended on implementing SK6812RGBW but they are, obviously, RGBW, not GRBW. Which are used in this product: https://www.adafruit.com/product/2837

I've made the library ignorant of color channels, so if your neopixels are GRB then you'll need to pixels[0] = (g, r, b). This allows me to easily support any configuration including pixels[0] = (r, g, b, w).

I've also implemented slicing, so you can do something like this pixels[0:2] = [(r,g,b), (r,g,b)] or even, pixels[:] = [(r,g,b), (r,g,b)] and the two colors will be repeated for the entire slice.

I've also made the library much faster, so the code might look a little more bogus. To get the performance (from ~500ms update time down to 100ms) I had to cut corners on the code cleanliness.

In this process, Pixels.set is gone, as I had to inline the function call for speed. Just use the setters pixels[]

I've also seperated the test code and implementation a little better, so you can cut everything off below the Pixels class without concern.

Do you know what part number your strips are? I intended on implementing SK6812RGBW but they are, obviously, RGBW, not GRBW. Which are used in this product: https://www.adafruit.com/product/2837

I have an SK6812 strip and it's GRBW.
By the way, there may exist some memory issue: When using a modified version of RAINBOW() (GRBW, with a 0 representing White_Subpixel = 0 added to each member of rainbow list ) and still 60 LEDs, the micropython environment throws exceptions often:

Error for allocating memory: 15xxx bytes.

Current board: ESP-WROOM-32, micropython 1.12.

For a 300-LED strip(I assembled a panel), such memory consumption may boom the precious 100-ish KB of total available memory. Do you have a plan to optimize the memory usage in possible future versions?

Can you show me how you're using the Pixel library so I can verify there isn't anything unusual.

The library should work with any parts provided that the timings are the same, just make sure you increase the pixel_channels to 4 in the constructor. The tuples will be in the order that the strip expects, in your case (G, R, B, W)

You could try removing all references to self.colors as a memory saving strategy. It is only used for getting the pixel color tuples back out, if you've no need to do something like color = pixels[0] then it's a waste of memory.

I can investigate memory management but I suspect your problem is memory fragmentation, not overall memory availability. The WROOM should have plenty memory to spare though I don't know how large your code is.

A future memory optimisation would be to have this implemented in C, then I don't need to store lists and tuples, but instead a flat buffer. I could also consider using a bytearray which would offer some improvements. It's not on my calendar to do this anytime soon, and as for your case, your system should have more than 1.5K free memory.

Sorry I just woke up and noticed that I misread your comment, you're having trouble allocating 15k memory, and that is a large block of memory.

You can try calling micropython.mem_info(1) to see what your memory looks like before you initialise the Pixels library. Perhaps post that here too.

When running RAINBOW(), ESP32 will raise MemoryError with 60+ LEDs in RGB(bpp=3) mode(I tried in unmodified .py file), and the memory usage before importing Pixel lib is shown below:

00000: h=ShhBMhhDhhhh=Shh==============================================
00400: ================================================================
00800: ================================================================
00c00: ================================================================
01000: =========================hh==BhBShShh==h=======h=====h==SBhBhTBh
01400: Sh==h===========..Sh==h==h==hh====.....h=.....h=................
01800: ................h====...........................................
       (101 lines all free)
1b000: ....................................

Also, if there were some way to flatten the memory, one single sequence for 60LED * 3bpp strip will still have 1.5KB-ish memory consumption.

As the purpose of such an RMT implementation is, in my opinion, solve the flickering issue for large-scale strips(happens in CPU-based bit-banging) as well as putting the write() work in daemon to save precious CPU time for micropython interpreter(is it possible?), is there any chance to implement a generator which converts the default NeoPixel bytearray(see self.buf in This link, it's an ESP8266 bit-banging implementation but usable on esp32) to RMT-sequences in small batches since a 10-KB of [7,16, 14,12, 14,12, 14,12, 7,16,......](I mean 0b01110......) is way too luxurious as RAM is really limited.

Also, if there were some way to flatten the memory, one single sequence for 60LED * 3bpp strip will still have 1.5KB-ish memory consumption

Flattening the memory out of Python objects down to a buffer is what I want to do as early as realistically possible, because that's a 10x saving from your 15k to 1.5k, though 60 LED at 24bit/LED should be 180 bytes.

is there any chance to implement a generator which converts the default NeoPixel bytearray ... to RMT-sequences in small batches

This is on the tables, but currently I'm waiting for streaming to land in the RMT library of MicroPython. I cannot write partial sequences back to back, the timing is too sensitive and there will be large jitter/defects. The RMT hardware supports a ping-pong buffer (active buffer, next buffer) approach I believe, or at least a means of streaming the data in. That is an option, provided that MicroPython can generate the colors fast enough. I will try implement this when the RMT module supports it and we will only know then if it can be done well.

RAM being limited is very much the nature of embedded, and every byte of RAM not used is a waste. But, obviously, we can't be blowing the RAM budget. I'm not happy with using the apparent 15k of memory, I'll try reduce that. 60 LEDs should only use 180 bytes when stored as a pixel buffer. There is the unfortunate case that this grows when using RMT to approximately twice as large since the on time and off time need to be specified. There are proposals for alternate API that may improve this in the future.

I have an ESP32 Devkit v1 available, which has a WROOM. I'll test my own setup with this (instead of the TinyPICO) and get that memory usage down.

There are ways to reduce the memory footprint. One of these would be to defer the building of the pixels array from the update to the write method. What's more, one could replace the += loop by a list comprehension: rmt.write_pulses accepts lists as well as tuples in the current implementation. The write method would look like:

    def write(self):
        # The bus should be idle low ( I think... )
        # So we finish low and start high.
        pulses = [clocks for bit in (D_ONE if ((channel >> bit) & 1) else D_ZERO  for colors in self.colors for channel in colors for bit in range(CHANNEL_WIDTH - 1, -1, -1)) for clocks in bit]
        pulses[-1] = T_RST
        self.rmt.write_pulses(pulses, start=1)

The original rainbow test failed with an out of memory error on my ESP32Pico, but, with the modification, every call to write only eats about 20K memory, which is released as soon as the write method exits.

By the way, every pixel needs a minimum of 3x8x2=48 integers (two per bit, 3 channels and 8bit resolution), so, for the rainbow test one needs to create a list/tuple of 2880 integers...

so, for the rainbow test one needs to create a list/tuple of 2880 integers...

'Yes', the calculation on integer numbers is correct, and each integer takes 2 bytes: According to RMT's documentation (v4.0-rc was used since 'latest' version seemed to have that image removed), each bit in LED strip data structure will have to consume a whopping 32 bits(4 bytes) of memory. If programs were to construct a full RMT-compatible array and output it(hopefully in daemon), the memory usage for such a 60-LED-RGB strip would be 5760 bytes(and for a 300-GRBW strip which I'm building, unfortunately, 38400bytes), still kind of high.

But anyway, hopes that more experienced developers could come out with some solutions balancing the memory and execution time, or something smaller than rmt_item32_t could be implemented.

You've both touched on a very obvious point which is that RMT is a memory hog. I'll try @fstengel's fix, every bit of saving counts. Best case scenario the entire buffer wouldn't need to be generated ahead of time, instead it should be shifted in as its consumed. That's what FastLED does, and any sane implementation should do the same. For now, I can't do that with the MicroPython implementation of RMT.

I don't think you will ever be able to do that with Python. For 150 pixels the calculation of pulses with @fstengels generator expression took 50 ms, so a live calculation is not an option. Maybe you can still write it a little bit faster (I guess the expression does a lot of copying and no preallocation), but I don't think you can keep up with the RMT. I outsourced the pulse calculation to a C module because of that.

In-time pulse calculation is done for example in the FastLED library - but using this implementation breaks the current RMT implementation, because it defines its own interrupt handlers. However this is the only way to ever have a memory-efficient and fast driver. I'm working with PSRAM, so memory is not a problem for me...

My intent was still to have all the colours in a python buffer before writing, but this can be made significantly smaller as they can be compacted in a bytearray as just RGB values and not rmt items.

It would then just be memory copying while streaming. I do not know if it will work though. I'm just hopeful.

I couldn't get rid of timing glitches when using the current RMT implementation. I don't know why, but apparently the interrupt copying from buffer to RMT memory couldn't catch up all the time. Increasing the RMT memory helped a bit but couldn't resolve it completely.

I ended up "porting" the FastLED driver to C. Haven't had any glitches so far. It's blazingly fast and uses very little RAM. You can do animations with 200 Hz refresh rate if you get the rendering done in time...

@carstenblt

Can you please share the final firmware you are integrating in esp32 in which you have ported FastLed driver to C.
(Micropython Firmware)

@tech-shubham
You can find it here: https://github.com/carstenblt/micropython
Some details on how to enable it and why the code looks like crap: micropython/micropython#5623 (comment)

In my own codebase I modified it a bit further and let it use the full RMT memory (therefore you can only use 1 RMT instead of 8 - FastLED had the ability to run several neopixel strips in parallel, but I never fully implemented that anyways). I had another glitch after several hours of running but that modification should make it even more stable.

@carstenblt

Thank you so much for your support.

The thing is I just need a favour if you can compile it and share with me .bin file as I have my windows laptop and somehow its not compiling through esp idf tools.

If you share the .bin file then I can directly put it into my esp32 board.

Your efforts are much appreciated, thanks.

@carstenblt

Can you please share the compiled .bin file which I can use it as a bootloader.

@carstenblt

Makefile shows multiple problem while compiling. It shows this error and number of more errors.
"Makefile:427: *** target pattern contains no `%'. Stop."

Kindly Please help out.

@carstenblt

I did able to compile and run finally.

But still the problem remains, in my code I am running it for 64 Leds and there is still random flickering.
But I appreciate your efforts.

@tech-shubham Are you sure you are using neopixel2? I did not touch the original neopixel library.

Yes, I miss something previously.
But,Thanks @carstenblt now its working just fine.

Just a problem is, we can't run two led strips simultaneously with this update.

Yep... The interrupt handler could handle multiple strips, it would just need some more work.

Well, I will be waiting for your next update. Kindly do tell me when its done.

Thanks!

@carstenblt

Does your micropython Firmware: https://github.com/carstenblt/micropython
Supports esp32 + psRam, I mean to ask either we have to change some settings or does it support external 4MB(or above) SPIRAM.

Thanks for this, solved my flickering neopixels on ESP32.

@tech-shubham, @vjdw Any chance that you can share a compiled version of the firmware that you got working? 🙏 Or even better, share instructions of what changes are necessary to build...

Sorry @martinjo, I'm only using a short strip and so all I needed was the original pixels.py gist. Maybe @tech-shubham can help with the firmware.