001package com.github.sarxos.webcam.ds.gstreamer;
002
003import java.awt.Dimension;
004import java.awt.image.BufferedImage;
005import java.awt.image.DataBufferInt;
006import java.io.File;
007import java.nio.IntBuffer;
008import java.util.ArrayList;
009import java.util.List;
010import java.util.concurrent.TimeUnit;
011import java.util.concurrent.atomic.AtomicBoolean;
012
013import org.bridj.Platform;
014import org.gstreamer.Caps;
015import org.gstreamer.Element;
016import org.gstreamer.ElementFactory;
017import org.gstreamer.Pad;
018import org.gstreamer.Pipeline;
019import org.gstreamer.State;
020import org.gstreamer.Structure;
021import org.gstreamer.elements.RGBDataSink;
022import org.slf4j.Logger;
023import org.slf4j.LoggerFactory;
024
025import com.github.sarxos.webcam.WebcamDevice;
026import com.github.sarxos.webcam.WebcamResolution;
027
028
029public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, WebcamDevice.FPSSource {
030
031        /**
032         * Logger.
033         */
034        private static final Logger LOG = LoggerFactory.getLogger(GStreamerDevice.class);
035
036        /**
037         * Limit the lateness of frames to no more than 20ms (half a frame at 25fps)
038         */
039        private static final long LATENESS = 20; // ms
040
041        /**
042         * Video format to capture.
043         */
044        private static final String FORMAT_MIME = "video/x-raw-yuv";
045
046        /**
047         * All possible resolutions - populated while initialization phase.
048         */
049        private Dimension[] resolutions = null;
050
051        /**
052         * Device name, immutable. Used only on Windows platform.
053         */
054        private final String name;
055        /**
056         * Device name, immutable. Used only on Linux platform.
057         */
058        private final File vfile;
059
060        /* gstreamer stuff */
061
062        private Pipeline pipe = null;
063        private Element source = null;
064        private Element filter = null;
065        private RGBDataSink sink = null;
066
067        private Caps caps = null;
068
069        /* logic */
070
071        private AtomicBoolean open = new AtomicBoolean(false);
072        private AtomicBoolean disposed = new AtomicBoolean(false);
073        private AtomicBoolean starting = new AtomicBoolean(false);
074        private AtomicBoolean initialized = new AtomicBoolean(false);
075        private Dimension resolution = WebcamResolution.VGA.getSize();
076        private BufferedImage image = null;
077
078        /* used to calculate fps */
079
080        private long t1 = -1;
081        private long t2 = -1;
082
083        private volatile double fps = 0;
084
085        /**
086         * Create GStreamer webcam device.
087         * 
088         * @param name the name of webcam device
089         */
090        protected GStreamerDevice(String name) {
091                this.name = name;
092                this.vfile = null;
093        }
094
095        protected GStreamerDevice(File vfile) {
096                this.name = null;
097                this.vfile = vfile;
098        }
099
100        /**
101         * Initialize webcam device.
102         */
103        private synchronized void init() {
104
105                if (!initialized.compareAndSet(false, true)) {
106                        return;
107                }
108
109                LOG.debug("GStreamer webcam device initialization");
110
111                pipe = new Pipeline(name);
112
113                if (Platform.isWindows()) {
114                        source = ElementFactory.make("dshowvideosrc", "source");
115                        source.set("device-name", name);
116                } else if (Platform.isLinux()) {
117                        source = ElementFactory.make("v4l2src", "source");
118                        source.set("device", vfile.getAbsolutePath());
119                }
120
121                sink = new RGBDataSink(name, this);
122                sink.setPassDirectBuffer(true);
123                sink.getSinkElement().setMaximumLateness(LATENESS, TimeUnit.MILLISECONDS);
124                sink.getSinkElement().setQOSEnabled(true);
125
126                filter = ElementFactory.make("capsfilter", "filter");
127
128                if (Platform.isLinux()) {
129                        pipe.addMany(source, filter, sink);
130                        Element.linkMany(source, filter, sink);
131                        pipe.setState(State.READY);
132                }
133
134                resolutions = parseResolutions(source.getPads().get(0));
135
136                if (Platform.isLinux()) {
137                        pipe.setState(State.NULL);
138                        Element.unlinkMany(source, filter, sink);
139                        pipe.removeMany(source, filter, sink);
140                }
141        }
142
143        /**
144         * Use GStreamer to get all possible resolutions.
145         * 
146         * @param pad the pad to get resolutions from
147         * @return Array of resolutions supported by device connected with pad
148         */
149        private static final Dimension[] parseResolutions(Pad pad) {
150
151                List<Dimension> dimensions = new ArrayList<Dimension>();
152
153                Caps caps = pad.getCaps();
154
155                Structure structure = null;
156                String mime = null;
157
158                int n = caps.size();
159                int i = 0;
160
161                int w = -1;
162                int h = -1;
163
164                do {
165
166                        structure = caps.getStructure(i++);
167
168                        LOG.debug("Found format structure {}", structure);
169
170                        mime = structure.getName();
171
172                        if (mime.equals(FORMAT_MIME)) {
173                                if (Platform.isWindows()) {
174                                        w = structure.getRange("width").getMinInt();
175                                        h = structure.getRange("height").getMinInt();
176                                        dimensions.add(new Dimension(w, h));
177                                } else if (Platform.isLinux()) {
178                                        if ("YUY2".equals(structure.getFourccString("format"))) {
179                                                w = structure.getInteger("width");
180                                                h = structure.getInteger("height");
181                                                dimensions.add(new Dimension(w, h));
182                                        }
183                                }
184                        }
185
186                } while (i < n);
187
188                return dimensions.toArray(new Dimension[dimensions.size()]);
189        }
190
191        @Override
192        public String getName() {
193                if (Platform.isWindows()) {
194                        return name;
195                } else if (Platform.isLinux()) {
196                        return vfile.getAbsolutePath();
197                } else {
198                        throw new RuntimeException("Platform not supported by GStreamer capture driver");
199                }
200        }
201
202        @Override
203        public Dimension[] getResolutions() {
204                init();
205                return resolutions;
206        }
207
208        @Override
209        public Dimension getResolution() {
210                return resolution;
211        }
212
213        @Override
214        public void setResolution(Dimension size) {
215                this.resolution = size;
216        }
217
218        @Override
219        public BufferedImage getImage() {
220                return image;
221        }
222
223        @Override
224        public void open() {
225
226                if (!open.compareAndSet(false, true)) {
227                        return;
228                }
229
230                LOG.debug("Opening GStreamer device");
231
232                init();
233
234                starting.set(true);
235
236                Dimension size = getResolution();
237
238                image = new BufferedImage(size.width, size.height, BufferedImage.TYPE_INT_RGB);
239                image.setAccelerationPriority(0);
240                image.flush();
241
242                if (caps != null) {
243                        caps.dispose();
244                }
245
246                caps = Caps.fromString(String.format("%s,width=%d,height=%d", FORMAT_MIME, size.width, size.height));
247
248                filter.setCaps(caps);
249
250                LOG.debug("Link elements");
251
252                pipe.addMany(source, filter, sink);
253                Element.linkMany(source, filter, sink);
254                pipe.setState(State.PLAYING);
255
256                // wait max 20s for image to appear
257                synchronized (this) {
258                        LOG.debug("Wait for device to be ready");
259                        try {
260                                this.wait(20000);
261                        } catch (InterruptedException e) {
262                                return;
263                        }
264                }
265        }
266
267        @Override
268        public void close() {
269
270                if (!open.compareAndSet(true, false)) {
271                        return;
272                }
273
274                LOG.debug("Closing GStreamer device");
275
276                image = null;
277
278                LOG.debug("Unlink elements");
279
280                pipe.setState(State.NULL);
281                Element.unlinkMany(source, filter, sink);
282                pipe.removeMany(source, filter, sink);
283        }
284
285        @Override
286        public void dispose() {
287
288                if (!disposed.compareAndSet(false, true)) {
289                        return;
290                }
291
292                LOG.debug("Disposing GStreamer device");
293
294                close();
295
296                filter.dispose();
297                source.dispose();
298                sink.dispose();
299                pipe.dispose();
300                caps.dispose();
301        }
302
303        @Override
304        public boolean isOpen() {
305                return open.get();
306        }
307
308        @Override
309        public void rgbFrame(boolean preroll, int width, int height, IntBuffer rgb) {
310
311                LOG.trace("New RGB frame");
312
313                if (t1 == -1 || t2 == -1) {
314                        t1 = System.currentTimeMillis();
315                        t2 = System.currentTimeMillis();
316                }
317
318                BufferedImage tmp = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
319                tmp.setAccelerationPriority(0);
320                tmp.flush();
321
322                rgb.get(((DataBufferInt) tmp.getRaster().getDataBuffer()).getData(), 0, width * height);
323
324                image = tmp;
325
326                if (starting.compareAndSet(true, false)) {
327
328                        synchronized (this) {
329                                this.notifyAll();
330                        }
331
332                        LOG.debug("GStreamer device ready");
333                }
334
335                t1 = t2;
336                t2 = System.currentTimeMillis();
337
338                fps = (4 * fps + 1000 / (t2 - t1 + 1)) / 5;
339        }
340
341        @Override
342        public double getFPS() {
343                return fps;
344        }
345
346        public Pipeline getPipe() {
347                return pipe;
348        }
349
350        public Element getSource() {
351                return source;
352        }
353
354        public Element getFilter() {
355                return filter;
356        }
357
358        public RGBDataSink getSink() {
359                return sink;
360        }
361
362        public Caps getCaps() {
363                return caps;
364        }
365}