SDL Updating Texture issues

SDL Development
#1

Hello Everybody,

I was hoping that maybe somebody could provide me with some assistance. I
am receiving an H.264 video streaming and successfully decoding it with
FFMPEG. However, it can display the first frame of data but then after that
the screen never updates. It just appears to become a static image. I am
using YUV pixel format, and I am receiving it in that format as well. Also
I am using SDL_UpdateYUVTexture().

Here is my code:

int main()
{
WORD wVersionRequested;
WSADATA wsaData;
int wsaerr;

if (SDL_Init(SDL_INIT_EVERYTHING)) {
fprintf(stderr, “Could not initialize SDL - %s\n”, SDL_GetError());
exit(1);
}
// Using MAKEWORD macro, Winsock version request 2.2
wVersionRequested = MAKEWORD(2, 2);

wsaerr = WSAStartup(wVersionRequested, &wsaData);

if (wsaerr != 0)
{
/* Tell the user that we could not find a usable /
/ WinSock DLL.*/
printf(“The Winsock dll not found!\n”);
return 0;
}
else
{
printf(“The Winsock dll found!\n”);
printf(“The status: %s.\n”, wsaData.szSystemStatus);
}

/* Confirm that the WinSock DLL supports 2.2./
/ Note that if the DLL supports versions greater /
/ than 2.2 in addition to 2.2, it will still return /
/ 2.2 in wVersion since that is the version we /
/ requested. /
if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
{
/ Tell the user that we could not find a usable /
/ WinSock DLL.*/
printf(“The dll do not support the Winsock version %u.%u!\n”,
LOBYTE(wsaData.wVersion), HIBYTE(wsaData.wVersion));
WSACleanup();
return 0;
}
else
{
printf(“The dll supports the Winsock version %u.%u!\n”,
LOBYTE(wsaData.wVersion), HIBYTE(wsaData.wVersion));
printf(“The highest version this dll can support: %u.%u\n”,
LOBYTE(wsaData.wHighVersion), HIBYTE(wsaData.wHighVersion));
}

ULONG localif;
/INT Ret;
HANDLE ThreadHandle;
DWORD ThreadId;
WSAEVENT AcceptEvent;
char buf[1024];
int buflen = 1024, rc, err;/

SOCKET s;
SOCKET ns;
SOCKADDR_IN multi, safrom;
int fromlen;

int totalSize = 0;

AVCodec *codec;
AVCodecContext *codecContext;
int frame;
int got_picture;
AVFrame picture;
AVPacket packet;
SwsContext convertContext;
uint16_t i = 1;
//std::queue queue;
//std::list list;
AVCodecParserContext *parser;
std::vector<uint8_t> buffer;
//moodycamel::ConcurrentQueue protoQueue;

SDL_Window *window;
SDL_Renderer *renderer;
SDL_Texture *bmp;
SDL_Rect rect;

file.open(“log.txt”);

s = socket(AF_INET, SOCK_STREAM, IPPROTO_RM);

multi.sin_family = AF_INET;
multi.sin_port = htons(5150);
multi.sin_addr.s_addr = inet_addr(“234.5.6.7”);
int bindResult = bind(s, (PSOCKADDR)&multi, sizeof(multi));

if (bindResult < 0)
{
std::cout << "bindResult: " << WSAGetLastError() << std::endl;
}

listen(s, 10);

//if ((AcceptEvent = WSACreateEvent()) == WSA_INVALID_EVENT)
//{
// printf(“WSACreateEvent() failed with error %d\n”, WSAGetLastError());
// return 1;
//}
//else
// printf(“WSACreateEvent() is OK!\n”);

//// Create a worker thread to service completed I/O requests
//if ((ThreadHandle = CreateThread(NULL, 0, WorkerThread,
(LPVOID)AcceptEvent, 0, &ThreadId)) == NULL)
//{
// printf(“CreateThread() failed with error %d\n”, GetLastError());
// return 1;
//}
//else
// printf(“CreateThread() should be fine!\n”);

localif = inet_addr(“192.168.1.2”);
setsockopt(s, IPPROTO_RM, RM_ADD_RECEIVE_IF, (char *)&localif,
sizeof(localif));

fromlen = sizeof(safrom);
ns = accept(s, (SOCKADDR *)&safrom, &fromlen);

closesocket(s); // Don’t need to listen anymore

std::string received;

av_register_all();

int horizontal = 0;
int vertical = 0;

GetDesktopResolution(horizontal, vertical);

codec = avcodec_find_decoder(CODEC_ID_H264);
if (!codec) {
std::cout << “codec not found” << std::endl;
std::cin.get();
}

codecContext = avcodec_alloc_context3(codec);

/if (codec->capabilities & CODEC_CAP_TRUNCATED)
codecContext->flags |= CODEC_FLAG_TRUNCATED;/

//codecContext->flags |= CODEC_FLAG_LOW_DELAY;
codecContext->flags2 |= CODEC_FLAG2_CHUNKS;

codecContext->width = horizontal;
codecContext->height = vertical;
codecContext->codec_id = CODEC_ID_H264;
codecContext->codec_type = AVMEDIA_TYPE_VIDEO;
codecContext->pix_fmt = PIX_FMT_YUV420P;
codecContext->thread_type = 0;

if (avcodec_open2(codecContext, codec, NULL) < 0) {
std::cout << “could not open codec” << std::endl;
std::cin.get();
}

convertContext = sws_getContext(
codecContext->width,
codecContext->height,
PIX_FMT_RGB32,
codecContext->width,
codecContext->height,
PIX_FMT_YUV420P,
SWS_BICUBIC,
NULL,
NULL,
NULL
);

parser = av_parser_init(CODEC_ID_H264);

picture = av_frame_alloc();

if (ns == INVALID_SOCKET)
{
std::cout << “accept didn’t work!” << std::endl;
std::cin.get();
}

/if (WSASetEvent(AcceptEvent) == FALSE)
{
printf(“WSASetEvent() failed with error %d\n”, WSAGetLastError());
return 1;
}
else
printf(“WSASetEvent() should be working!\n”);/

window = SDL_CreateWindow(“YUV”, SDL_WINDOWPOS_UNDEFINED,
SDL_WINDOWPOS_UNDEFINED, codecContext->width, codecContext->height,
SDL_WINDOW_SHOWN);
renderer = SDL_CreateRenderer(window, -1, 0);
bmp = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV,
SDL_TEXTUREACCESS_STREAMING, codecContext->width, codecContext->height);

//receive = SDL_CreateThread(receiveThread, “ReceiveThread”, (void *)NULL);
bool quit = false;

rect.x = 0;
rect.y = 0;
rect.w = codecContext->width;
rect.h = codecContext->height;

while (!quit)
{
while (true)
{
MadProto proto;
int result = recvfrom(ns, (char *)&proto, sizeof(MadProto), 0, (struct
sockaddr *)&multi, &fromlen);

if (result < 0)
{
std::cout << "receive failed! error: " << WSAGetLastError() << std::endl;
break;
}
else
{
std::cout << “receive successful, received " << result << " bytes” <<
std::endl;

if (ntohs(proto.frame_end) == 1)
{
uint8_t *outbuffer = NULL;
int outBufSize = 0;
int rc = av_parser_parse2(parser, codecContext, &outbuffer, &outBufSize,
buffer.data(), buffer.size(), 0, 0, 0);

if (outBufSize <= 0)
{
std::cout << “parsing failed!” << std::endl;
std::cout << "outBufSize: " << outBufSize << std::endl;
break;
}

if (rc)
{
std::cout << "rc: " << rc << std::endl;
std::cout << “parsing successful!” << std::endl;
//std::cin.get();

av_init_packet(&packet);
packet.size = outBufSize;
packet.data = outbuffer;

frame = avcodec_decode_video2(codecContext, picture, &got_picture, &packet);

if (frame < 0)
{
std::cout << “decoding was unsuccessful!” << std::endl;
break;
}

if (got_picture)
{
std::cout << “decoding was successful!” << std::endl;
std::cout << "decoded length was: " << frame << std::endl;
buffer.empty();

//std::cin.get();
int code = SDL_UpdateYUVTexture(bmp, NULL, picture->data[0],
picture->linesize[0],
picture->data[1], picture->linesize[1],
picture->data[2], picture->linesize[2]);

if (code < 0)
{
std::cout << "unable to update texture " << SDL_GetError() << std::endl;
std::cin.get();
}

code = SDL_RenderClear(renderer);

if (code < 0)
{
std::cout << "renderer clear failed " << SDL_GetError() << std::endl;
std::cin.get();
}

code = SDL_RenderCopy(renderer, bmp, NULL, &rect);

if (code < 0)
{
std::cout << "renderer copy failed " << SDL_GetError() << std::endl;
std::cin.get();
}

SDL_RenderPresent(renderer);

SDL_Delay(40);
}

av_free_packet(&packet);
}
}
else
{
std::copy(proto.payload, proto.payload + ntohs(proto.nal_length),
std::back_inserter(buffer));
std::cout << “frame is continuing!” << std::endl;

//queue.push(proto);
//list.push_front(proto);
}
}
}

SDL_WaitEvent(&event);
switch (event.type)
{
case SDL_QUIT:
quit = true;
break;
}
}

std::cout << “closing everything!” << std::endl;
av_frame_free(&picture);
closesocket(ns);
fclose(f);
std::cin.get();
return 0;
}

I would appreciate any advice and assistance given.

Thanks,

Ryan

#2

Any updates on this?

#3

Can you make the project (code + executable) available for download ?

#4

Client:

Server:

I have made my client available, you will need both if you plan on running the client successfully. The exe is all that is available for the server. I will post the code for it if it needed, but everything should be coming down correctly. These projects are built on Windows and use PGM for communication. If you have any issues running the code, please let me know.

Again, I appreciate any assistance given.

#5

1 - Replace SDL_WaitEvent() with a while (SDL_PollEvent(…))
2 - You cannot wait 40msec (SDL_Delay(40) in the end of the display code),
also if u wanna go 25fps, you should give network and decoder time to work,
so you should adapt the period you wait to the time you took to receive and
display frame.On Sun, Jun 14, 2015 at 10:56 PM, madprogrammer24 <r.bauer234 at gmail.com> wrote:

Client:
https://www.dropbox.com/s/ttk46srhfjxrzld/MadLink_client.zip?dl=0

Server:
https://www.dropbox.com/s/n4m29b4rkc44u7e/Debug.zip?dl=0

I have made my client available, you will need both if you plan on running
the client successfully. The exe is all that is available for the server. I
will post the code for it if it needed, but everything should be coming
down correctly. These projects are built on Windows and use PGM for
communication. If you have any issues running the code, please let me know.

Again, I appreciate any assistance given.

SDL mailing list
SDL at lists.libsdl.org
http://lists.libsdl.org/listinfo.cgi/sdl-libsdl.org

–
Ing. Gabriele Greco, DARTS Engineering
Tel: +39-0100980150 Fax: +39-0100980184
s-mail: Piazza Della Vittoria 9/3 - 16121 GENOVA (ITALY)

#6

You cannot wait 40msec (SDL_Delay(40) in the end of the display code), also if u wanna go 25fps, you should give network and decoder time to work, so you should adapt the period you wait to the time you took to receive and display frame.

I am not entirely sure how to adapt it? Do you mean by adding some SDL timers around my network and decoder code then change my delay accordingly?

#7

If u wanna go 25fps you can for instance:

1 get the timestamp when you decode the first frame, and sum to it 40msec.
2 receive data from network and decode the frame
3 before displaying the next frame wait for the difference between the
current timestamp and your stored one.
4 after displaying the frame sum 40msec to your stored timestamp
5 go to 2

This will give you 25fps on capable enough hardware, if you have slow
network or hardware that is just enough to decode the stream you’ll have
better performance splitting the network handling, the decoding process and
the frame display in different threads, but this will need some extra care
to call the SDL functions only from the right thread.On Mon, Jun 15, 2015 at 3:32 PM, madprogrammer24 <r.bauer234 at gmail.com> wrote:

Quote:

You cannot wait 40msec (SDL_Delay(40) in the end of the display code),
also if u wanna go 25fps, you should give network and decoder time to work,
so you should adapt the period you wait to the time you took to receive and
display frame.

I am not entirely sure how to adapt it? Do you mean by adding some SDL
timers around my network and decoder code then change my delay accordingly?

SDL mailing list
SDL at lists.libsdl.org
http://lists.libsdl.org/listinfo.cgi/sdl-libsdl.org

–
Ing. Gabriele Greco, DARTS Engineering
Tel: +39-0100980150 Fax: +39-0100980184
s-mail: Piazza Della Vittoria 9/3 - 16121 GENOVA (ITALY)

#8

Okay, I tried using the techniques described in the previous post and from one of the tutorials at Lazy Foo. Namely, this one: http://lazyfoo.net/tutorials/SDL/22_timing/index.php

I am still not seeing the screen refresh at all. Could this be caused by me using the same system for the server and client? I am just doing that for testing purposes.

I will probably download the source for the Lazy Foo tutorial and mess with it, so I can understand what is going on better.

Code:

//Screen dimension constants
const int SCREEN_WIDTH = horizontal;
const int SCREEN_HEIGHT = vertical;
const int SCREEN_FPS = 60;
const int SCREEN_TICKS_PER_FRAME = 1000 / SCREEN_FPS;

//The frames per second timer
LTimer fpsTimer;

//The frames per second cap timer
LTimer capTimer;

//Start counting frames per second
int countedFrames = 0;
fpsTimer.start();

while (true)
{
	//Uint32 startTime = SDL_GetTicks();
	//Start cap timer
	capTimer.start();

	while (SDL_PollEvent(&event))
	{
		switch (event.type)
		{
		case SDL_QUIT:
			SDL_Quit();
			break;
		}
	}

	MadProto proto;
	int result = recvfrom(ns, (char *)&proto, sizeof(MadProto), 0, (struct sockaddr *)&multi, &fromlen);

	//Calculate and correct fps
	float avgFPS = countedFrames / (fpsTimer.getTicks() / 1000.f);
	if (avgFPS > 2000000)
	{
		avgFPS = 0;
	}

	if (result < 0)
	{
		std::cout << "receive failed! error: " << WSAGetLastError() << std::endl;
		break;
	}
	else
	{
		std::cout << "receive successful, received " << result << " bytes" << std::endl;

		if (ntohs(proto.frame_end) == 1)
		{
			uint8_t *outbuffer = NULL;
			int outBufSize = 0;
			int rc = av_parser_parse2(parser, codecContext, &outbuffer, &outBufSize, buffer.data(), buffer.size(), 0, 0, 0);

			if (outBufSize <= 0)
			{
				std::cout << "parsing failed!" << std::endl;
				std::cout << "outBufSize: " << outBufSize << std::endl;
			}

			if (rc)
			{
				++countedFrames;

				//If frame finished early
				int frameTicks = capTimer.getTicks();
				if (frameTicks < SCREEN_TICKS_PER_FRAME)
				{
					//Wait remaining time
					SDL_Delay(SCREEN_TICKS_PER_FRAME - frameTicks);
				}

				std::cout << "rc: " << rc << std::endl;
				std::cout << "parsing successful!" << std::endl;
				//std::cin.get();

				av_init_packet(&packet);
				packet.size = outBufSize;
				packet.data = outbuffer;

				frame = avcodec_decode_video2(codecContext, picture, &got_picture, &packet);

				if (frame < 0)
				{
					std::cout << "decoding was unsuccessful!" << std::endl;
					break;
				}

				if (got_picture)
				{
					unsigned char *pixels = NULL;
					int pitch;

					std::cout << "decoding was successful!" << std::endl;
					std::cout << "decoded length was: " << frame << std::endl;
					buffer.empty();

					int code = 0;

					code = SDL_UpdateYUVTexture(bmp, NULL, picture->data[0], picture->linesize[0],
						picture->data[1], picture->linesize[1],
						picture->data[2], picture->linesize[2]);

					SDL_RenderClear(renderer);
					SDL_RenderCopy(renderer, bmp, NULL, NULL);
					SDL_RenderPresent(renderer);
				}

				av_free_packet(&packet);
			}
		}
		else
		{
			std::copy(proto.payload, proto.payload + ntohs(proto.nal_length), std::back_inserter(buffer));
			std::cout << "frame is continuing!" << std::endl;

			//queue.push(proto);
			//list.push_front(proto);
		}
	}
}
#9

Would be worth checking to see if SDL_UpdateYUVTexture fails

#10

MrTAToad wrote:

Would be worth checking to see if SDL_UpdateYUVTexture fails

I thought about that and put that into my code, after my latest post. It wasn’t failing, kind of wish it was… it would at least give me something different to look at.

#11

I figured out my problem, and I thought I should update this issue for anybody else who has something similar. My issue was I didn’t loop around my av_parser_parse2() call as the documentation instructed. Once I did that, I was able to receive all data that was parsed and my screen started updating correctly. I appreciate everybody’s help on helping me solve this issue.

Now on to making my stream more stable and improve latency… woohoo!

Parsing loop below:

Code:

while (dataSize)
{
int rc = av_parser_parse2(parser, codecContext, &outbuffer, &outBufSize, frameData, dataSize, 0, 0, 0);

if (outBufSize <= 0)
{
	std::cout << "parsing failed!" << std::endl;
	std::cout << "outBufSize: " << outBufSize << std::endl;
}

frameData += rc;
dataSize -= rc;

if (outBufSize)
{
	++countedFrames;

	std::cout << "rc: " << rc << std::endl;
	std::cout << "parsing successful!" << std::endl;
	//std::cin.get();

	av_init_packet(&packet);
	packet.size = outBufSize;
	packet.data = outbuffer;

	frame = avcodec_decode_video2(codecContext, picture, &got_picture, &packet);

	if (frame < 0)
	{
		std::cout << "decoding was unsuccessful!" << std::endl;
		break;
	}

	if (got_picture)
	{
		unsigned char *pixels = NULL;
		int pitch;

		std::cout << "decoding was successful!" << std::endl;
		std::cout << "decoded length was: " << frame << std::endl;
		buffer.empty();

		int code = 0;

		code = SDL_UpdateYUVTexture(bmp, NULL, picture->data[0], picture->linesize[0],
			picture->data[1], picture->linesize[1],
			picture->data[2], picture->linesize[2]);

		if (code < 0)
		{
			std::cout << "updating failed! " << SDL_GetError() << std::endl;
			std::cin.get();
		}							

		//If frame finished early
		int frameTicks = capTimer.getTicks();
		if (frameTicks < SCREEN_TICKS_PER_FRAME)
		{
			//Wait remaining time
			SDL_Delay(SCREEN_TICKS_PER_FRAME - frameTicks);
		}
	}

	av_free_packet(&packet);
}

}

// Render Clear, Render Copy, Render Present after the loop